10_transfer-images-http.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transferring images from the Postgres DB into the Digital Ocean Space\n",
    "\n",
    "Here's what we need to do to transfer over the raw image data so it's no longer in the database.\n",
    "\n",
    "1. Get a list of all images from the API\n",
    "2. Download all images into a temp folder, with the name determined from an image hash\n",
    "3. Sanity & integrity checks\n",
    "4. Upload folder to Digital Ocean Space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. list of all images to query\n",
    "\n",
    "I have node running the server.js file in the api folder on my test server, which has the same database as prod."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No, I don't want to implement log levels\n",
    "VERBOSE_MODE=False\n",
    "SHORT=False\n",
    "RANGE=(None,None)\n",
    "# RANGE=(0,100)\n",
    "# RANGE=(100,None)\n",
    "num_worker_threads = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import io\n",
    "import json\n",
    "import os\n",
    "import requests\n",
    "from PIL import Image\n",
    "import random\n",
    "import imagehash\n",
    "import queue\n",
    "import threading\n",
    "import time\n",
    "import imghdr\n",
    "import os\n",
    "import ipyplot\n",
    "from collections import defaultdict\n",
    "\n",
    "# base = 'http://159.89.80.47'\n",
    "base = 'http://api.firewallcafe.com'\n",
    "folder = 'images'\n",
    "\n",
    "try:\n",
    "    os.mkdir(folder)\n",
    "except FileExistsError:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('image_pages.json', 'r') as f:\n",
    "    image_pages = list(json.loads(f.read()).values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "361676 161223\n"
     ]
    }
   ],
   "source": [
    "api_image_urls = [img['image_href'] for img_page in image_pages for img in img_page]\n",
    "print(len(api_image_urls), len(set(api_image_urls)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Okay, got about .3 million images and about half of those are unique. Looks good, let's get 'em!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1a. image saving functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'9780f5773298c566.jpg'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def hash_image(img):\n",
    "    return imagehash.phash(img)\n",
    "img = Image.open(\"temp.jpg\")\n",
    "f'{hash_image(img)}.jpg'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _save_image(url):\n",
    "    # save image in temp folder\n",
    "    r = requests.get(url, timeout=15)\n",
    "    in_memory_file = io.BytesIO(r.content)\n",
    "    im = Image.open(in_memory_file)\n",
    "    hash_array = hash_image(im)\n",
    "    ext = im.format.lower()\n",
    "    fname = f'{folder}/{hash_array}.{ext}'\n",
    "    im.save(fname)\n",
    "    if VERBOSE_MODE:\n",
    "        print('saved', fname)\n",
    "    return f'{hash_array}.{ext}'\n",
    "# _save_image('https://i.kym-cdn.com/photos/images/newsfeed/000/008/330/funny_cat_pictures_144.jpg')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. download all images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "failures = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set up an object that can track the status of the overall job with inputs from all the threads."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Storage:\n",
    "    def __init__(self):\n",
    "        self.start = time.time()\n",
    "        self._lock = threading.Lock()\n",
    "        self.failures = {}\n",
    "        self.errors = {}\n",
    "        self.count = 0\n",
    "        self.url_to_hash = {}\n",
    "        \n",
    "    def add_failure(self, image_id, url, e):\n",
    "        with self._lock:\n",
    "            self.failures[image_id] = url\n",
    "            self.errors[image_id] = e\n",
    "            if VERBOSE_MODE:\n",
    "                print(\"error\", e)\n",
    "\n",
    "    def increment_count(self):\n",
    "        with self._lock:\n",
    "            self.count += 1\n",
    "            \n",
    "    def save_image(self, url):\n",
    "        fname = _save_image(url)\n",
    "        with self._lock:\n",
    "            self.url_to_hash[url] = fname\n",
    "            \n",
    "    def save_mapping(self):\n",
    "        fname = f'url_to_hash_{round(time.time())}.json'\n",
    "        with self._lock:\n",
    "            with open(fname, 'w') as f:\n",
    "                f.write(json.dumps(self.url_to_hash))\n",
    "        return fname\n",
    "    \n",
    "    def print_status(self):\n",
    "        with self._lock:\n",
    "            print(f\"processing {self.count} took {round((time.time()-self.start)/60, 1)} minutes. \\\n",
    "{round(100*(len(self.failures)/self.count), 1)}% failure rate & {len(self.failures)} failures, {round((time.time()-self.start)/self.count, 2)} s per, ts {time.time()}\")\n",
    "\n",
    "storage = Storage()\n",
    "def worker():\n",
    "    while True:\n",
    "        image_id, url = q.get()\n",
    "        if url is None:\n",
    "            print('x', end=' ')\n",
    "            q.task_done()\n",
    "            break\n",
    "        \n",
    "        storage.increment_count()\n",
    "        \n",
    "        try:\n",
    "            storage.save_image(url)\n",
    "        except Exception as e:\n",
    "            storage.add_failure(image_id, url, e)\n",
    "        print_freq = 100\n",
    "        if VERBOSE_MODE:\n",
    "            print_freq = 10\n",
    "        if not SHORT:\n",
    "            print_freq = 500\n",
    "        if random.randint(0,print_freq) == 0:\n",
    "            storage.print_status()\n",
    "        q.task_done()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This... is gonna take a while."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "done with /image queries\n",
      "telling threads to stop\n",
      "waiting\n",
      "processing 719 took 0.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122497.980612\n",
      "processing 1022 took 0.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122501.0673604\n",
      "processing 2082 took 0.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122511.5273967\n",
      "processing 2489 took 0.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122515.3634527\n",
      "processing 2599 took 0.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122516.4390888\n",
      "processing 2636 took 0.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122516.7991254\n",
      "processing 2686 took 0.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122517.284828\n",
      "processing 2772 took 0.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122518.1385815\n",
      "processing 3578 took 0.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122526.2459843\n",
      "processing 3684 took 0.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122527.305179\n",
      "processing 3813 took 0.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122528.619664\n",
      "processing 3925 took 0.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122529.7695892\n",
      "processing 4454 took 0.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122534.8624058\n",
      "processing 5090 took 0.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122541.1072721\n",
      "processing 5170 took 0.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122541.8642478\n",
      "processing 5935 took 1.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122549.2443044\n",
      "processing 6081 took 1.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122550.721355\n",
      "processing 6878 took 1.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122558.5524206\n",
      "processing 6973 took 1.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122559.4599926\n",
      "processing 7591 took 1.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122566.001304\n",
      "processing 8800 took 1.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122578.292114\n",
      "processing 8998 took 1.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122580.3950067\n",
      "processing 9633 took 1.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122587.272708\n",
      "processing 10002 took 1.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122591.3044424\n",
      "processing 10401 took 1.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122595.5525985\n",
      "processing 10640 took 1.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122597.8424768\n",
      "processing 11310 took 1.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122604.4852862\n",
      "processing 11530 took 1.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122606.7437596\n",
      "processing 11651 took 2.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122607.9061623\n",
      "processing 11889 took 2.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122610.6298811\n",
      "processing 12231 took 2.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122614.1610615\n",
      "processing 12416 took 2.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122616.1487474\n",
      "processing 12651 took 2.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122618.6481378\n",
      "processing 12812 took 2.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122620.2428737\n",
      "processing 13215 took 2.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122624.3597302\n",
      "processing 13636 took 2.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122628.8996983\n",
      "processing 13742 took 2.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122629.9678283\n",
      "processing 14023 took 2.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122632.9568374\n",
      "processing 14752 took 2.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122640.4582965\n",
      "processing 15079 took 2.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122643.5619996\n",
      "processing 16027 took 2.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122653.1788418\n",
      "processing 16640 took 2.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122659.3512838\n",
      "processing 16708 took 2.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122659.9716253\n",
      "processing 16757 took 2.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122660.4363823\n",
      "processing 17647 took 3.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122669.2613006\n",
      "processing 17941 took 3.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122672.1590664\n",
      "processing 18464 took 3.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122677.1088333\n",
      "processing 18883 took 3.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122681.407853\n",
      "processing 20514 took 3.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122697.9945421\n",
      "processing 20893 took 3.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122701.9350076\n",
      "processing 21195 took 3.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122705.0466886\n",
      "processing 21205 took 3.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122705.1165028\n",
      "processing 21503 took 3.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122708.2681026\n",
      "processing 21801 took 3.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122711.8570418\n",
      "processing 21925 took 3.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122713.0987208\n",
      "processing 22139 took 3.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122715.3417244\n",
      "processing 22788 took 3.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122721.98747\n",
      "processing 22824 took 3.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122722.309609\n",
      "processing 23470 took 4.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122729.7313192\n",
      "processing 23628 took 4.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122731.3469987\n",
      "processing 23951 took 4.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122735.2465749\n",
      "processing 24545 took 4.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122741.344782\n",
      "processing 24897 took 4.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122745.2962441\n",
      "processing 25085 took 4.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122747.1592631\n",
      "processing 25813 took 4.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122754.615331\n",
      "processing 26127 took 4.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122757.6013486\n",
      "processing 26432 took 4.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122760.7828426\n",
      "processing 26985 took 4.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122766.3470001\n",
      "processing 27079 took 4.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122767.3473253\n",
      "processing 27321 took 4.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122769.7000675\n",
      "processing 27655 took 4.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122773.0251787\n",
      "processing 28302 took 4.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122779.4834259\n",
      "processing 29232 took 5.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122789.03141\n",
      "processing 29326 took 5.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122789.9958332\n",
      "processing 29431 took 5.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122790.9931657\n",
      "processing 30460 took 5.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122802.1913595\n",
      "processing 30988 took 5.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122807.538067\n",
      "processing 31805 took 5.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122815.8563638\n",
      "processing 32394 took 5.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122821.7715485\n",
      "processing 32881 took 5.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122826.984124\n",
      "processing 33419 took 5.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122832.5238593\n",
      "processing 33557 took 5.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122834.0178657\n",
      "processing 33730 took 5.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122835.7497633\n",
      "processing 34748 took 5.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122846.145083\n",
      "processing 35483 took 6.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122853.5393472\n",
      "processing 36074 took 6.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122859.4246132\n",
      "processing 36134 took 6.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122860.1247416\n",
      "processing 36245 took 6.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122861.3654249\n",
      "processing 36469 took 6.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122863.7006931\n",
      "processing 36532 took 6.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122864.3250237\n",
      "processing 37707 took 6.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122875.5942369\n",
      "processing 37942 took 6.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122877.958429\n",
      "processing 37996 took 6.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122878.573785\n",
      "processing 38452 took 6.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122883.924984\n",
      "processing 38848 took 6.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122888.4050593\n",
      "processing 39283 took 6.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122892.7201915\n",
      "processing 39440 took 6.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122894.3867364\n",
      "processing 40411 took 6.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122904.0080376\n",
      "processing 40622 took 6.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122906.1034353\n",
      "processing 42392 took 7.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122924.9189346\n",
      "processing 43083 took 7.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122932.179524\n",
      "processing 43350 took 7.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122934.787614\n",
      "processing 43985 took 7.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122941.572533\n",
      "processing 44456 took 7.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122946.5662837\n",
      "processing 44770 took 7.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122949.8420923\n",
      "processing 44809 took 7.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122950.1841779\n",
      "processing 45247 took 7.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122954.7609432\n",
      "processing 45487 took 7.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122957.2782128\n",
      "processing 46039 took 7.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122962.9186468\n",
      "processing 46656 took 8.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122969.3768942\n",
      "processing 47611 took 8.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122979.2506466\n",
      "processing 48343 took 8.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122986.8939507\n",
      "processing 48606 took 8.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122989.71042\n",
      "processing 49454 took 8.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618122998.89241\n",
      "processing 49687 took 8.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123001.3573616\n",
      "processing 49758 took 8.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123002.1422627\n",
      "processing 50213 took 8.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123007.135939\n",
      "processing 50658 took 8.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123011.6429205\n",
      "processing 50883 took 8.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123014.1686795\n",
      "processing 51040 took 8.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123015.7155685\n",
      "processing 51383 took 8.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123019.1394844\n",
      "processing 51407 took 8.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123019.367874\n",
      "processing 51636 took 8.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123021.5590165\n",
      "processing 52413 took 9.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123030.2497833\n",
      "processing 52698 took 9.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123033.1106453\n",
      "processing 53547 took 9.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123041.612352\n",
      "processing 53559 took 9.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123041.717072\n",
      "processing 53677 took 9.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123042.844059\n",
      "processing 53698 took 9.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123043.0794303\n",
      "processing 54127 took 9.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123047.2029426\n",
      "processing 54437 took 9.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123050.3350787\n",
      "processing 54700 took 9.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123052.78852\n",
      "processing 55653 took 9.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123062.4936278\n",
      "processing 55798 took 9.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123063.9088721\n",
      "processing 56392 took 9.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123069.9901516\n",
      "processing 57266 took 9.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123078.4825358\n",
      "processing 57317 took 9.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123079.0350592\n",
      "processing 57742 took 9.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123083.2226126\n",
      "processing 58278 took 10.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123088.4771059\n",
      "processing 58547 took 10.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123091.0931113\n",
      "processing 59432 took 10.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123100.295862\n",
      "processing 59524 took 10.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123101.2413359\n",
      "processing 59981 took 10.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123105.9408057\n",
      "processing 60623 took 10.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123112.4185061\n",
      "processing 61378 took 10.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123120.584188\n",
      "processing 61673 took 10.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123123.4047127\n",
      "processing 61947 took 10.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123126.2690563\n",
      "processing 62188 took 10.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123128.7155154\n",
      "processing 62952 took 10.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123136.840814\n",
      "processing 63176 took 10.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123139.1795616\n",
      "processing 63630 took 10.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123143.8012722\n",
      "processing 63815 took 10.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123145.677766\n",
      "processing 63911 took 10.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123146.5424547\n",
      "processing 64442 took 11.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123152.971325\n",
      "processing 65214 took 11.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123161.1798925\n",
      "processing 65483 took 11.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123164.0851264\n",
      "processing 65630 took 11.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123165.5586967\n",
      "processing 65685 took 11.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123166.120195\n",
      "processing 65814 took 11.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123167.425216\n",
      "processing 66432 took 11.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123173.4069126\n",
      "processing 67799 took 11.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123187.431207\n",
      "processing 67925 took 11.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123188.7133179\n",
      "processing 67957 took 11.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123189.0384495\n",
      "processing 68123 took 11.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123190.7279685\n",
      "processing 68596 took 11.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123196.1331863\n",
      "processing 69113 took 11.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123201.0441096\n",
      "processing 69237 took 11.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123202.2139826\n",
      "processing 70299 took 12.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123213.2200978\n",
      "processing 71186 took 12.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123221.9508271\n",
      "processing 71456 took 12.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123224.8291326\n",
      "processing 72399 took 12.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123234.8049958\n",
      "processing 73348 took 12.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123243.9592278\n",
      "processing 73843 took 12.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123249.521432\n",
      "processing 73921 took 12.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123250.3412395\n",
      "processing 73961 took 12.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123250.7152395\n",
      "processing 74132 took 12.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123252.3290687\n",
      "processing 74305 took 12.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123254.0384924\n",
      "processing 74378 took 12.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123254.843341\n",
      "processing 74603 took 12.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123257.0873413\n",
      "processing 75581 took 13.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123267.5683224\n",
      "processing 76279 took 13.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123275.4060555\n",
      "processing 78171 took 13.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123295.2395835\n",
      "processing 78289 took 13.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123296.3925014\n",
      "processing 78958 took 13.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123303.7732463\n",
      "processing 79098 took 13.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123305.1889987\n",
      "processing 79441 took 13.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123308.5081244\n",
      "processing 80298 took 13.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123316.81443\n",
      "processing 81735 took 14.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123331.6717591\n",
      "processing 82592 took 14.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123340.8901358\n",
      "processing 82642 took 14.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123341.4247062\n",
      "processing 82827 took 14.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123343.2767556\n",
      "processing 83034 took 14.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123345.5237486\n",
      "processing 83080 took 14.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123346.0039744\n",
      "processing 83687 took 14.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123352.002937\n",
      "processing 84084 took 14.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123356.2281547\n",
      "processing 85628 took 14.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123371.8420684\n",
      "processing 86695 took 14.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123382.8481593\n",
      "processing 87011 took 14.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123385.9025052\n",
      "processing 87243 took 15.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123388.5155194\n",
      "processing 87637 took 15.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123392.2884586\n",
      "processing 88216 took 15.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123398.561714\n",
      "processing 88854 took 15.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123404.9287174\n",
      "processing 89046 took 15.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123406.8550806\n",
      "processing 91688 took 15.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123435.0220597\n",
      "processing 92223 took 15.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123440.196227\n",
      "processing 92524 took 15.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123443.111945\n",
      "processing 93001 took 16.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123447.975456\n",
      "processing 93116 took 16.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123449.166295\n",
      "processing 93262 took 16.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123450.5999749\n",
      "processing 94435 took 16.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123463.191887\n",
      "processing 95320 took 16.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123472.119635\n",
      "processing 95593 took 16.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123474.7835147\n",
      "processing 96217 took 16.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123482.0183067\n",
      "processing 97537 took 16.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123495.5126739\n",
      "processing 97782 took 16.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123498.0539632\n",
      "processing 97905 took 16.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123499.392416\n",
      "processing 99864 took 17.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123520.443622\n",
      "processing 100397 took 17.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123526.0546832\n",
      "processing 100829 took 17.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123530.9012983\n",
      "processing 101099 took 17.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123533.627137\n",
      "processing 102034 took 17.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123543.5446239\n",
      "processing 102133 took 17.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123544.67959\n",
      "processing 103785 took 17.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123562.2116678\n",
      "processing 103998 took 17.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123564.4686322\n",
      "processing 104914 took 18.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123573.7748358\n",
      "processing 105140 took 18.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123576.5100353\n",
      "processing 105593 took 18.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123581.2454274\n",
      "processing 106281 took 18.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123588.7725353\n",
      "processing 106382 took 18.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123589.9813044\n",
      "processing 107004 took 18.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123597.2115726\n",
      "processing 107344 took 18.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123600.6504092\n",
      "processing 107369 took 18.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123600.8967507\n",
      "processing 107565 took 18.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123602.889025\n",
      "processing 107998 took 18.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123607.733407\n",
      "processing 108194 took 18.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123609.829322\n",
      "processing 108204 took 18.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123609.9978714\n",
      "processing 108676 took 18.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123614.7796278\n",
      "processing 109005 took 18.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123618.0838592\n",
      "processing 109595 took 18.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123624.5630505\n",
      "processing 111148 took 19.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123641.111462\n",
      "processing 112450 took 19.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123654.8877006\n",
      "processing 112515 took 19.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123655.5611017\n",
      "processing 112997 took 19.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123660.7432725\n",
      "processing 113030 took 19.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123661.2813492\n",
      "processing 113062 took 19.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123661.5735674\n",
      "processing 113562 took 19.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123667.1591804\n",
      "processing 114274 took 19.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123675.5507467\n",
      "processing 114624 took 19.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123679.617389\n",
      "processing 114755 took 19.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123681.0455704\n",
      "processing 115090 took 19.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123685.3909535\n",
      "processing 115748 took 20.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123692.6912699\n",
      "processing 117034 took 20.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123706.6404192\n",
      "processing 117615 took 20.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123713.4518135\n",
      "processing 118268 took 20.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123720.9619937\n",
      "processing 118367 took 20.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123721.9983437\n",
      "processing 119974 took 20.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123739.3993726\n",
      "processing 120592 took 20.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123746.2742622\n",
      "processing 121025 took 21.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123750.9913032\n",
      "processing 121947 took 21.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123760.6610112\n",
      "processing 122021 took 21.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123761.5159924\n",
      "processing 122580 took 21.3 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123767.6019487\n",
      "processing 123527 took 21.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123777.9346242\n",
      "processing 123688 took 21.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123779.7353954\n",
      "processing 124753 took 21.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123791.3677788\n",
      "processing 125589 took 21.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123800.155157\n",
      "processing 125692 took 21.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123801.223868\n",
      "processing 125730 took 21.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123801.8407412\n",
      "processing 125896 took 21.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123803.5339277\n",
      "processing 126927 took 22.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123814.5749943\n",
      "processing 127489 took 22.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123820.6019683\n",
      "processing 127705 took 22.2 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123822.8770196\n",
      "processing 128623 took 22.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123832.621286\n",
      "processing 128832 took 22.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123834.7275596\n",
      "processing 129311 took 22.5 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123839.6837564\n",
      "processing 129930 took 22.6 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123846.1427364\n",
      "processing 130372 took 22.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123850.8840623\n",
      "processing 130431 took 22.7 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123851.4459622\n",
      "processing 130832 took 22.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123855.736554\n",
      "processing 130962 took 22.8 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123857.0231862\n",
      "processing 131629 took 22.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123863.8792996\n",
      "processing 131708 took 22.9 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123864.7679808\n",
      "processing 131936 took 23.0 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123867.1887028\n",
      "processing 132854 took 23.1 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123876.8079412\n",
      "processing 134398 took 23.4 minutes. 0.0% failure rate & 0 failures, 0.01 s per, ts 1618123892.8480635\n",
      "processing 134755 took 23.4 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123896.3472009\n",
      "processing 134788 took 23.4 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123896.6663482\n",
      "processing 135593 took 23.6 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123904.8953776\n",
      "processing 135676 took 23.6 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123905.8134365\n",
      "processing 136117 took 23.7 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123910.710399\n",
      "processing 136579 took 23.8 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123915.3654907\n",
      "processing 136663 took 23.8 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123916.150393\n",
      "processing 136727 took 23.8 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123916.7582808\n",
      "processing 136766 took 23.8 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123917.162712\n",
      "processing 137056 took 23.8 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123920.1996183\n",
      "processing 137638 took 23.9 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123926.647453\n",
      "processing 137703 took 24.0 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123927.3116767\n",
      "processing 138295 took 24.1 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123933.6682725\n",
      "processing 139438 took 24.3 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123946.03875\n",
      "processing 140251 took 24.4 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123954.3286185\n",
      "processing 140495 took 24.4 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123956.9435465\n",
      "processing 141583 took 24.7 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123969.7713604\n",
      "processing 141589 took 24.7 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123969.853318\n",
      "processing 142281 took 24.8 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618123976.996725\n",
      "processing 144629 took 25.2 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124002.99947\n",
      "processing 144669 took 25.2 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124003.3782036\n",
      "processing 144786 took 25.2 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124004.5180461\n",
      "processing 144824 took 25.2 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124005.037221\n",
      "processing 145762 took 25.4 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124016.3615599\n",
      "processing 145833 took 25.4 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124017.1069415\n",
      "processing 147107 took 25.7 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124030.3757608\n",
      "processing 147122 took 25.7 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124030.5244153\n",
      "processing 147324 took 25.7 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124032.665485\n",
      "processing 147351 took 25.7 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124033.0025842\n",
      "processing 147561 took 25.8 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124035.7613583\n",
      "processing 148571 took 25.9 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124046.1605613\n",
      "processing 148954 took 26.0 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124050.5210767\n",
      "processing 149482 took 26.1 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124056.7646687\n",
      "processing 150605 took 26.3 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124069.3226302\n",
      "processing 151359 took 26.4 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124077.1020327\n",
      "processing 151561 took 26.5 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124079.136086\n",
      "processing 151772 took 26.5 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124081.940554\n",
      "processing 152310 took 26.6 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124087.6295264\n",
      "processing 152413 took 26.6 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124088.7739577\n",
      "processing 152418 took 26.6 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124088.8188384\n",
      "processing 153077 took 26.8 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124095.5757344\n",
      "processing 153510 took 26.8 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124099.9085774\n",
      "processing 153757 took 26.9 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124102.4835224\n",
      "processing 154162 took 26.9 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124106.6500103\n",
      "processing 155005 took 27.1 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124115.8660436\n",
      "processing 155023 took 27.1 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124116.018063\n",
      "processing 155618 took 27.2 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124122.001487\n",
      "processing 156311 took 27.3 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124129.6782897\n",
      "processing 156527 took 27.4 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124131.8437936\n",
      "processing 156572 took 27.4 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124132.262192\n",
      "processing 156586 took 27.4 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124132.471789\n",
      "processing 157195 took 27.5 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124138.9415443\n",
      "processing 157307 took 27.5 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124140.5815399\n",
      "processing 159786 took 28.0 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124169.6490374\n",
      "processing 159812 took 28.0 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124169.955218\n",
      "processing 159866 took 28.0 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124170.6214366\n",
      "processing 160114 took 28.1 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124173.8366308\n",
      "processing 160223 took 28.1 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124175.576297\n",
      "processing 160404 took 28.1 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124177.7421916\n",
      "processing 161428 took 28.3 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124190.3108933\n",
      "processing 162391 took 28.5 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124202.1900475\n",
      "processing 163392 took 28.7 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124214.3272278\n",
      "processing 163428 took 28.7 minutes. 0.0% failure rate & 1 failures, 0.01 s per, ts 1618124214.6604242\n",
      "processing 164272 took 29.0 minutes. 0.0% failure rate & 5 failures, 0.01 s per, ts 1618124231.2576547\n",
      "processing 164297 took 29.2 minutes. 0.0% failure rate & 9 failures, 0.01 s per, ts 1618124244.3614812\n",
      "processing 164358 took 29.4 minutes. 0.0% failure rate & 17 failures, 0.01 s per, ts 1618124253.1051526\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rowan\\anaconda3\\lib\\site-packages\\PIL\\Image.py:960: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing 164499 took 30.2 minutes. 0.0% failure rate & 33 failures, 0.01 s per, ts 1618124301.4435785\n",
      "processing 165283 took 32.4 minutes. 0.1% failure rate & 145 failures, 0.01 s per, ts 1618124435.5448728\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rowan\\anaconda3\\lib\\site-packages\\PIL\\Image.py:2929: UserWarning: image file could not be identified because WEBP support not installed\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing 166465 took 36.2 minutes. 0.2% failure rate & 330 failures, 0.01 s per, ts 1618124663.1704326\n",
      "processing 166600 took 36.5 minutes. 0.2% failure rate & 350 failures, 0.01 s per, ts 1618124682.841308\n",
      "processing 167009 took 37.8 minutes. 0.2% failure rate & 413 failures, 0.01 s per, ts 1618124758.7031612\n",
      "processing 168084 took 40.8 minutes. 0.3% failure rate & 579 failures, 0.01 s per, ts 1618124935.4604037\n",
      "processing 168223 took 41.3 minutes. 0.4% failure rate & 598 failures, 0.01 s per, ts 1618124966.3299487\n",
      "processing 169561 took 46.8 minutes. 0.5% failure rate & 840 failures, 0.02 s per, ts 1618125296.775748\n",
      "processing 169764 took 47.6 minutes. 0.5% failure rate & 870 failures, 0.02 s per, ts 1618125345.5458918\n",
      "processing 169922 took 48.1 minutes. 0.5% failure rate & 899 failures, 0.02 s per, ts 1618125375.2273517\n",
      "processing 169952 took 48.2 minutes. 0.5% failure rate & 903 failures, 0.02 s per, ts 1618125380.6328995\n",
      "processing 171092 took 51.0 minutes. 0.7% failure rate & 1114 failures, 0.02 s per, ts 1618125549.8800745\n",
      "processing 171582 took 51.4 minutes. 0.7% failure rate & 1198 failures, 0.02 s per, ts 1618125573.2703104\n",
      "processing 171968 took 51.6 minutes. 0.7% failure rate & 1287 failures, 0.02 s per, ts 1618125587.9875326\n",
      "processing 172353 took 52.1 minutes. 0.8% failure rate & 1335 failures, 0.02 s per, ts 1618125614.362344\n",
      "processing 172368 took 52.1 minutes. 0.8% failure rate & 1335 failures, 0.02 s per, ts 1618125615.2953634\n",
      "processing 173018 took 54.0 minutes. 0.8% failure rate & 1434 failures, 0.02 s per, ts 1618125733.0090566\n",
      "processing 173213 took 54.7 minutes. 0.9% failure rate & 1473 failures, 0.02 s per, ts 1618125771.6280177\n",
      "processing 174059 took 57.5 minutes. 0.9% failure rate & 1601 failures, 0.02 s per, ts 1618125938.7421992\n",
      "processing 174313 took 58.8 minutes. 0.9% failure rate & 1638 failures, 0.02 s per, ts 1618126016.0824947\n",
      "processing 174649 took 59.9 minutes. 1.0% failure rate & 1686 failures, 0.02 s per, ts 1618126083.7451055\n",
      "processing 174804 took 60.1 minutes. 1.0% failure rate & 1708 failures, 0.02 s per, ts 1618126099.0048604\n",
      "processing 174913 took 60.6 minutes. 1.0% failure rate & 1730 failures, 0.02 s per, ts 1618126125.8345277\n",
      "processing 175123 took 61.3 minutes. 1.0% failure rate & 1764 failures, 0.02 s per, ts 1618126165.7565758\n",
      "processing 176837 took 66.5 minutes. 1.1% failure rate & 2010 failures, 0.02 s per, ts 1618126479.8309898\n",
      "processing 177872 took 68.7 minutes. 1.2% failure rate & 2139 failures, 0.02 s per, ts 1618126614.4060426\n",
      "processing 178271 took 69.8 minutes. 1.2% failure rate & 2185 failures, 0.02 s per, ts 1618126676.6905434\n",
      "processing 178674 took 71.3 minutes. 1.2% failure rate & 2222 failures, 0.02 s per, ts 1618126767.3230026\n",
      "processing 179468 took 74.4 minutes. 1.3% failure rate & 2338 failures, 0.02 s per, ts 1618126952.285127\n",
      "processing 179470 took 74.4 minutes. 1.3% failure rate & 2338 failures, 0.02 s per, ts 1618126952.5683699\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rowan\\anaconda3\\lib\\site-packages\\PIL\\TiffImagePlugin.py:763: UserWarning: Possibly corrupt EXIF data.  Expecting to read 34 bytes but only got 20. Skipping tag 42036\n",
      "  warnings.warn(\n",
      "C:\\Users\\rowan\\anaconda3\\lib\\site-packages\\PIL\\TiffImagePlugin.py:785: UserWarning: Corrupt EXIF data.  Expecting to read 4 bytes but only got 0. \n",
      "  warnings.warn(str(msg))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing 180692 took 78.0 minutes. 1.4% failure rate & 2509 failures, 0.03 s per, ts 1618127167.2367122\n",
      "processing 180993 took 78.8 minutes. 1.4% failure rate & 2562 failures, 0.03 s per, ts 1618127216.0320628\n",
      "processing 181151 took 79.3 minutes. 1.4% failure rate & 2595 failures, 0.03 s per, ts 1618127247.3494384\n",
      "processing 181470 took 80.6 minutes. 1.5% failure rate & 2638 failures, 0.03 s per, ts 1618127324.3446307\n",
      "processing 182029 took 82.1 minutes. 1.5% failure rate & 2717 failures, 0.03 s per, ts 1618127415.869392\n",
      "processing 182500 took 83.3 minutes. 1.5% failure rate & 2786 failures, 0.03 s per, ts 1618127487.7040124\n",
      "processing 183656 took 87.0 minutes. 1.6% failure rate & 2958 failures, 0.03 s per, ts 1618127711.8735435\n",
      "processing 183807 took 87.6 minutes. 1.6% failure rate & 2991 failures, 0.03 s per, ts 1618127748.9652636\n",
      "processing 184859 took 91.7 minutes. 1.7% failure rate & 3188 failures, 0.03 s per, ts 1618127989.7493973\n",
      "processing 185423 took 93.7 minutes. 1.8% failure rate & 3274 failures, 0.03 s per, ts 1618128112.881683\n",
      "processing 186601 took 96.6 minutes. 1.9% failure rate & 3474 failures, 0.03 s per, ts 1618128286.3585577\n",
      "processing 188279 took 101.9 minutes. 2.0% failure rate & 3761 failures, 0.03 s per, ts 1618128604.0948706\n",
      "processing 189045 took 104.9 minutes. 2.1% failure rate & 3900 failures, 0.03 s per, ts 1618128781.894295\n",
      "processing 189176 took 105.1 minutes. 2.1% failure rate & 3931 failures, 0.03 s per, ts 1618128796.1088593\n",
      "processing 189845 took 106.4 minutes. 2.1% failure rate & 4028 failures, 0.03 s per, ts 1618128874.559026\n",
      "processing 190320 took 107.2 minutes. 2.2% failure rate & 4093 failures, 0.03 s per, ts 1618128923.226638\n",
      "processing 190615 took 108.0 minutes. 2.2% failure rate & 4140 failures, 0.03 s per, ts 1618128970.2532032\n",
      "processing 190623 took 108.0 minutes. 2.2% failure rate & 4140 failures, 0.03 s per, ts 1618128970.3449585\n",
      "processing 190956 took 109.1 minutes. 2.2% failure rate & 4207 failures, 0.03 s per, ts 1618129035.4594376\n",
      "processing 191429 took 110.0 minutes. 2.2% failure rate & 4264 failures, 0.03 s per, ts 1618129088.1244771\n",
      "processing 191463 took 110.0 minutes. 2.2% failure rate & 4271 failures, 0.03 s per, ts 1618129091.2092302\n",
      "processing 191661 took 110.6 minutes. 2.2% failure rate & 4307 failures, 0.03 s per, ts 1618129126.9372308\n",
      "processing 192552 took 113.8 minutes. 2.3% failure rate & 4461 failures, 0.04 s per, ts 1618129317.9051044\n",
      "processing 192603 took 113.9 minutes. 2.3% failure rate & 4467 failures, 0.04 s per, ts 1618129321.7384279\n",
      "processing 192727 took 114.2 minutes. 2.3% failure rate & 4482 failures, 0.04 s per, ts 1618129342.8435369\n",
      "processing 192995 took 115.1 minutes. 2.3% failure rate & 4518 failures, 0.04 s per, ts 1618129394.6741533\n",
      "processing 193151 took 115.4 minutes. 2.4% failure rate & 4542 failures, 0.04 s per, ts 1618129415.7369585\n",
      "processing 193963 took 117.5 minutes. 2.4% failure rate & 4672 failures, 0.04 s per, ts 1618129538.9516697\n",
      "processing 194145 took 117.7 minutes. 2.4% failure rate & 4702 failures, 0.04 s per, ts 1618129553.5712397\n",
      "processing 194447 took 118.8 minutes. 2.4% failure rate & 4751 failures, 0.04 s per, ts 1618129618.5558226\n",
      "processing 194720 took 119.6 minutes. 2.5% failure rate & 4792 failures, 0.04 s per, ts 1618129666.6724315\n",
      "processing 194888 took 120.2 minutes. 2.5% failure rate & 4816 failures, 0.04 s per, ts 1618129701.2356005\n",
      "processing 195147 took 121.2 minutes. 2.5% failure rate & 4871 failures, 0.04 s per, ts 1618129761.4976788\n",
      "processing 195393 took 121.8 minutes. 2.5% failure rate & 4913 failures, 0.04 s per, ts 1618129798.8873482\n",
      "processing 195865 took 123.4 minutes. 2.6% failure rate & 5029 failures, 0.04 s per, ts 1618129892.432979\n",
      "processing 197220 took 125.9 minutes. 2.7% failure rate & 5260 failures, 0.04 s per, ts 1618130046.8482893\n",
      "processing 197718 took 126.4 minutes. 2.7% failure rate & 5313 failures, 0.04 s per, ts 1618130076.98975\n",
      "processing 198435 took 127.4 minutes. 2.7% failure rate & 5383 failures, 0.04 s per, ts 1618130135.689867\n",
      "processing 199158 took 130.1 minutes. 2.8% failure rate & 5492 failures, 0.04 s per, ts 1618130293.5945075\n",
      "processing 199448 took 131.2 minutes. 2.8% failure rate & 5554 failures, 0.04 s per, ts 1618130360.1009293\n",
      "processing 199761 took 132.3 minutes. 2.8% failure rate & 5606 failures, 0.04 s per, ts 1618130426.641986\n",
      "processing 200488 took 133.7 minutes. 2.8% failure rate & 5713 failures, 0.04 s per, ts 1618130514.1562788\n",
      "processing 201367 took 136.4 minutes. 2.9% failure rate & 5852 failures, 0.04 s per, ts 1618130676.8820093\n",
      "processing 201795 took 137.7 minutes. 2.9% failure rate & 5930 failures, 0.04 s per, ts 1618130749.896807\n",
      "processing 202255 took 139.8 minutes. 3.0% failure rate & 6014 failures, 0.04 s per, ts 1618130876.114526\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rowan\\anaconda3\\lib\\site-packages\\PIL\\TiffImagePlugin.py:785: UserWarning: Corrupt EXIF data.  Expecting to read 12 bytes but only got 4. \n",
      "  warnings.warn(str(msg))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing 202515 took 140.5 minutes. 3.0% failure rate & 6075 failures, 0.04 s per, ts 1618130920.445903\n",
      "processing 202923 took 141.6 minutes. 3.0% failure rate & 6176 failures, 0.04 s per, ts 1618130986.7774377\n",
      "processing 203706 took 144.4 minutes. 3.1% failure rate & 6346 failures, 0.04 s per, ts 1618131151.3742428\n",
      "processing 203804 took 144.7 minutes. 3.1% failure rate & 6365 failures, 0.04 s per, ts 1618131173.3877654\n",
      "processing 204585 took 147.6 minutes. 3.2% failure rate & 6502 failures, 0.04 s per, ts 1618131344.7359045\n",
      "processing 205052 took 149.2 minutes. 3.2% failure rate & 6595 failures, 0.04 s per, ts 1618131439.5949085\n",
      "saved to url_to_hash_1618131496.json\n"
     ]
    }
   ],
   "source": [
    "# set up threading and queueing infra\n",
    "q = queue.Queue()\n",
    "threads = []\n",
    "for i in range(num_worker_threads):\n",
    "    t = threading.Thread(target=worker)\n",
    "    t.start()\n",
    "    threads.append(t)\n",
    "\n",
    "# iterating over all images in DB\n",
    "start_idx = 0\n",
    "if SHORT:\n",
    "    start_idx = random.randint(5,50)\n",
    "if RANGE[0] != None:\n",
    "    start_idx = RANGE[0]\n",
    "if RANGE[1] != None:\n",
    "    end_idx = RANGE[1]\n",
    "else:\n",
    "    end_idx = len(image_pages)\n",
    "for j in image_pages[start_idx:end_idx]:\n",
    "    for img in j:\n",
    "        count += 1\n",
    "        # make filename from url\n",
    "        if img['image_href'] is not None:\n",
    "            url = img['image_href']\n",
    "        elif img['wordpress_attachment_file_path'] is not None:\n",
    "            url = 'http://firewallcafe.com' + img['wordpress_attachment_file_path']\n",
    "        else:\n",
    "            continue\n",
    "        # put the url into the queue for the workers\n",
    "        q.put((img['image_id'], url))\n",
    "    if SHORT:\n",
    "        print(\"skipping the rest of the pages because this is a test\")\n",
    "        break\n",
    "print(\"\\ndone with /image queries\\ntelling threads to stop\")\n",
    "for i in range(num_worker_threads):\n",
    "    q.put((None,None))\n",
    "print('waiting')\n",
    "for t in threads:\n",
    "    t.join(timeout=3*60)\n",
    "mapping_fname = storage.save_mapping()\n",
    "print(\"saved to\", mapping_fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved to url_to_hash_1618157001.json\n",
      "processing 361676 took 575.3 minutes. 22.9% failure rate & 82903 failures, 0.1 s per, ts 1618157007.5260353\n"
     ]
    }
   ],
   "source": [
    "mapping_fname = storage.save_mapping()\n",
    "print(\"saved to\", mapping_fname)\n",
    "storage.print_status()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Sanity & integrity checks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check all image URLs in the list against their corresponding filenames in the folder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "101693"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(storage.url_to_hash)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_images = set(os.listdir(folder))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "94884"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(saved_images)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check if we were able to get all the images we need by comparing the files we have saved to the original URL, and making sure there is coverage there."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found 267437 (73.94%), missing 94239\n"
     ]
    }
   ],
   "source": [
    "found_count = 0\n",
    "missing = []\n",
    "saved_urls = set(storage.url_to_hash.keys())\n",
    "for image_page in image_pages:\n",
    "    for image in image_page:\n",
    "        if image['image_href'] in saved_urls:\n",
    "            found_count += 1\n",
    "        else:\n",
    "            missing.append(image)\n",
    "print(f\"found {found_count} ({round(100*found_count/(found_count+len(missing)), 2)}%), missing {len(missing)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "non_jpgs = [fname for fname in saved_images if fname[-4:] != '.jpg' and fname[-5:] != '.jpeg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.026021246996332365"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(non_jpgs)/len(saved_images)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sanity-check a random selection of images to make sure they correspond to files we actually have."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = random.sample(storage.url_to_hash.items(), 5)\n",
    "originals = [el[0] for el in sample]\n",
    "copies = [el[1] for el in sample]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">0</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://vignette3.wikia.nocookie.net/godfather/images/4/4d/Fidel_Castro.jpg/revision/latest?cb=20100415052925</h4><img src=\"https://vignette3.wikia.nocookie.net/godfather/images/4/4d/Fidel_Castro.jpg/revision/latest?cb=20100415052925\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">1</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">http://firewallcafe.com/wp-content/uploads/2016/02/google-tor-03.png</h4><img src=\"http://firewallcafe.com/wp-content/uploads/2016/02/google-tor-03.png\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">2</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://usercontent2.hubstatic.com/7175055.jpg</h4><img src=\"https://usercontent2.hubstatic.com/7175055.jpg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">3</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://pbs.twimg.com/media/C10Z08LWgAUkGoS.jpg</h4><img src=\"https://pbs.twimg.com/media/C10Z08LWgAUkGoS.jpg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">4</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">http://thumbs.dreamstime.com/b/%E6%8A%97%E8%AE%AE%E5%9C%A8%E9%A6%99%E6%B8%AF-98303062.jpg</h4><img src=\"http://thumbs.dreamstime.com/b/%E6%8A%97%E8%AE%AE%E5%9C%A8%E9%A6%99%E6%B8%AF-98303062.jpg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ipyplot.plot_images(originals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">0</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">images/b115466fe2598c7a.jpeg</h4><img src=\"images/b115466fe2598c7a.jpeg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">1</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">images/a31bd85a1e21b727.png</h4><img src=\"images/a31bd85a1e21b727.png\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">2</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">images/b6a0f159cb1a9d0d.jpeg</h4><img src=\"images/b6a0f159cb1a9d0d.jpeg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">3</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">images/a525379fc7e82858.jpeg</h4><img src=\"images/a525379fc7e82858.jpeg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">4</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">images/96942bacb93e0a8f.jpeg</h4><img src=\"images/96942bacb93e0a8f.jpeg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ipyplot.plot_images([f'{folder}/{fname}' for fname in copies])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That checks out. Let's look at the images which are most referred to in the database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_image_urls = [img['image_href'] for img_page in image_pages for img in img_page]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "361676"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(api_image_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['http://img2.imgtn.bdimg.com/it/u=392972713,3849171745&fm=21&gp=0.jpg',\n",
       " 'https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/c5d5da98476585c3.jpg',\n",
       " 'http://www.bohemia-apartments.com/wp-content/uploads/2014/09/warhol4.jpg',\n",
       " 'https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/b830876d838f8fc6.jpg',\n",
       " 'https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/8000000000000000.gif',\n",
       " 'https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/fbecd2c88e161992.jpg',\n",
       " None,\n",
       " 'https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/b503786f05f31e70.jpg',\n",
       " 'http://cdn.wegotthiscovered.com/wp-content/uploads/Silent-Hills-670x335.jpg',\n",
       " 'http://img4.imgtn.bdimg.com/it/u=3630604270,565219518&fm=21&gp=0.jpg']"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.sample(api_image_urls, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "161223"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(api_image_urls))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Which images are duplicated the most in the API's database? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "url_counts = defaultdict(int)\n",
    "for url in api_image_urls:\n",
    "    url_counts[url] += 1\n",
    "most_redundant_images = list(sorted(url_counts.items(), key=lambda el: el[1], reverse=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">14083</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/8000000000000000.gif</h4><img src=\"https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/8000000000000000.gif\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">12213</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">None</h4><img src=\"None\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">448</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/8336792bb76a145a.jpg</h4><img src=\"https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/8336792bb76a145a.jpg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">442</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/8000000000000000.jpg</h4><img src=\"https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/8000000000000000.jpg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">91</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/eb19a4b183e69a93.jpg</h4><img src=\"https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/eb19a4b183e69a93.jpg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">63</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/d5dca3aa6a489db0.jpg</h4><img src=\"https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/d5dca3aa6a489db0.jpg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">63</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/f9a00fd284dfc28d.jpg</h4><img src=\"https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/f9a00fd284dfc28d.jpg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">60</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/db48983d04a1fb9e.jpg</h4><img src=\"https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/db48983d04a1fb9e.jpg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">60</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/c74cb89b61349c76.jpg</h4><img src=\"https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/c74cb89b61349c76.jpg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div><div style=\"display: inline-block; width: 320px; vertical-align: top; text-align: center;\"><h4 style=\"font-size: 12px\">60</h4><h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 90%; word-wrap: break-word; white-space: normal;\">https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/ee919166673392cc.jpg</h4><img src=\"https://firewall-cafe-space.nyc3.digitaloceanspaces.com/images/hashed/ee919166673392cc.jpg\" style=\"margin: 1px; width: 300px; border: 2px solid #ddd;\"/></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ipyplot.plot_images([f'{el[0]}' for el in most_redundant_images], [el[1] for el in most_redundant_images[:10]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Upload to Digital Ocean Space\n",
    "\n",
    "Use [s3cmd](https://docs.digitalocean.com/products/spaces/resources/s3cmd/) and run the command for uploading an entire folder (see [documentation](https://docs.digitalocean.com/products/spaces/resources/s3cmd-usage/)): `s3cmd put * s3://firewall-cafe-space/images/hashes/ --recursive`"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}