bin/create_a_deep-learning_dataset_using_google-images.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将以下代码直接copy进相应Google Image页面的 Inspect-->Console中，urls.txt中保存了该页面中的全部图片链接\n",
    "  \n",
    "```\n",
    "// pull down jquery into the JavaScript console\n",
    "var script = document.createElement('script');\n",
    "script.src = \"https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js\";\n",
    "document.getElementsByTagName('head')[0].appendChild(script);\n",
    "// grab the URLs\n",
    "var urls = $('.rg_di .rg_meta').map(function() { return JSON.parse($(this).text()).ou; });\n",
    "```\n",
    "```\n",
    "// write the URls to file (one per line)\n",
    "var textToSave = urls.toArray().join('\\n');\n",
    "var hiddenElement = document.createElement('a');\n",
    "hiddenElement.href = 'data:attachment/text,' + encodeURI(textToSave);\n",
    "hiddenElement.target = '_blank';\n",
    "hiddenElement.download = 'urls.txt';\n",
    "hiddenElement.click();\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用下面的爬虫代码爬取urls.txt中全部的图片链接，将图片下载到本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "# download_images.py\n",
    "\n",
    "from imutils import paths\n",
    "import argparse\n",
    "import requests\n",
    "import cv2\n",
    "import os\n",
    "import time\n",
    "\n",
    "rows = open('urls.txt').read().strip().split(\"\\n\")\n",
    "total = 0\n",
    "\n",
    "# loop the URLs\n",
    "for url in rows:\n",
    "    try:\n",
    "        # try to download the image\n",
    "        print(str(url))\n",
    "        r = requests.get(url)\n",
    "        time.sleep(5)\n",
    "        # save the image to disk\n",
    "        p = os.path.sep.join([args[\"output\"], \"{}.jpg\".format(str(total).zfill(8))])\n",
    "        with open(p, 'wb') as f:\n",
    "            f.write(r.content)\n",
    "\n",
    "        # update the counter\n",
    "        print(\"[INFO] downloaded: %d\"%total)\n",
    "        total += 1\n",
    "        \n",
    "    # handle if any exceptions are thrown during the download process\n",
    "    except:\n",
    "        print(\"[INFO] error downloading {}...skipping\".format(p))\n",
    "\n",
    "# loop over the image paths we just downloaded\n",
    "for imagePath in paths.list_images(args[\"output\"]):\n",
    "    # initialize if the image should be deleted or not\n",
    "    delete = False\n",
    "\n",
    "    # try to load the image\n",
    "    try:\n",
    "        image = cv2.imread(imagePath)\n",
    "\n",
    "        # if the image is `None` then we could not properly load it\n",
    "        # from disk, so delete it\n",
    "        if image is None:\n",
    "            delete = True\n",
    "\n",
    "    # if OpenCV cannot load the image then the image is likely\n",
    "    # corrupt so we should delete it\n",
    "    except:\n",
    "        print(\"Except\")\n",
    "        delete = True\n",
    "\n",
    "    # check to see if the image should be deleted\n",
    "    if delete:\n",
    "        print(\"[INFO] deleting {}\".format(imagePath))\n",
    "        os.remove(imagePath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the script, make sure that the \"output file path\" is already existed\n",
    "$ python download_images.py --urls urls.txt --output images/santa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}