image_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "\n",
    "from selenium import webdriver\n",
    "\n",
    "import urllib.request"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting list of links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reached the end of page\n"
     ]
    }
   ],
   "source": [
    "#Selenium code to scroll to bottom of the page\n",
    "\n",
    "# search_query = \"jindo+dog\"\n",
    "search_query = \"shiba+dog\"\n",
    "\n",
    "link = \"https://www.google.com/search?q={}&tbm=isch\".format(search_query)\n",
    "DRIVER_PATH = '/usr/local/bin/chromedriver'\n",
    "\n",
    "driver = webdriver.Chrome(executable_path=DRIVER_PATH)\n",
    "driver.get(link)\n",
    "\n",
    "SCROLL_PAUSE_TIME = 2\n",
    "\n",
    "# Get scroll height\n",
    "last_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "\n",
    "while True:\n",
    "    # Scroll down to bottom\n",
    "    driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "\n",
    "    # Wait to load page\n",
    "    time.sleep(SCROLL_PAUSE_TIME)\n",
    "\n",
    "    # Calculate new scroll height and compare with last scroll height\n",
    "    new_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "    \n",
    "    if new_height == last_height:\n",
    "    #break #insert press load more\n",
    "        try:\n",
    "            element = driver.find_elements_by_class_name('mye4qd') #returns list\n",
    "            element[0].click()\n",
    "        except:\n",
    "            break\n",
    "    last_height = new_height\n",
    "    \n",
    "print(\"Reached the end of page\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Return the lists of urls using \"data-src\" and \"src\" anchors and merge them to fill None values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "def null_count(l):\n",
    "    #given a list l, find the number of null\n",
    "    null_count = 0\n",
    "    \n",
    "    for element in l:\n",
    "        if element == None:\n",
    "            null_count += 1\n",
    "            \n",
    "    return null_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "674"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#return a list clickable objects for each individual image\n",
    "image_links = driver.find_elements_by_class_name('rg_i.Q4LuWd')\n",
    "total = len(image_links)\n",
    "total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "#use the list of object to search for 'data-src' and 'src' anchors\n",
    "data_src_links = [image_links[i].get_attribute('data-src') for i in range(total)]\n",
    "src_links = [image_links[i].get_attribute('src') for i in range(total)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "137"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_src_null_count = null_count(data_src_links)\n",
    "data_src_null_count #117"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "214"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "src_null_count = null_count(src_links)\n",
    "src_null_count #153"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Since the \"data_src\" anchor has less missing, we will use \"src\" anchor's list to fill in \"data_src\"\n",
    "\n",
    "for i,element in enumerate(data_src_links):\n",
    "    if element == None:\n",
    "        data_src_links[i] = src_links[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Nulls: 0, Length: 674'"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check null count again\n",
    "\"Nulls: {}, Length: {}\".format(null_count(data_src_links), len(data_src_links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Downloading Images using urllib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Pathing for first scrape of jindo (Skip if SECOND run through)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/Rich/Desktop/chrome_driver/images/jindo\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir images\n",
    "!mkdir images/jindo\n",
    "!mkdir images/shiba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "# os.chdir('./images/jindo')\n",
    "os.chdir('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('./images/shiba')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Want to be in the jindo directory\n",
    "!pwd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b> Pathing for second scrap for shibas (Skip if FIRST run through)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('.images/shiba')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/Rich/Desktop/chrome_driver/images/shiba\r\n"
     ]
    }
   ],
   "source": [
    "#You want to be in the shiba directory\n",
    "!pwd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b> Actual Scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,link in enumerate(data_src_links):\n",
    "    \n",
    "    #change comment based on jindo or shiba\n",
    "#     name = 'jindo{}.png'.format(i) \n",
    "    name = 'shiba{}.png'.format(i)\n",
    "    \n",
    "    urllib.request.urlretrieve(link, name)\n",
    "    time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}