instagrab.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Author: Patrik Wikstrom (patrik.wikstrom@qut.edu.au)\n",
    "# Organisation: QUT Digital Media Research Centre 2017\n",
    "# Version: 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "the_item = \"catchoftheday\"\n",
    "max_scrape = 150\n",
    "\n",
    "insta_path = \"explore/tags/\"\n",
    "prefix = \"photos\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import base64\n",
    "from string import ascii_letters, digits, punctuation, printable\n",
    "from IPython.display import Image, display, HTML\n",
    "from time import sleep\n",
    "from collections import Counter\n",
    "from selenium import webdriver\n",
    "#from selenium.webdriver.common.keys import Keys\n",
    "from bs4 import BeautifulSoup\n",
    "from copy import copy\n",
    "import requests\n",
    "import json\n",
    "import os\n",
    "import platform\n",
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data/data_photos_catchoftheday\n",
      "data/media\n"
     ]
    }
   ],
   "source": [
    "# Create folders as necessary\n",
    "data_path = \"data/data\"  + \"_\" + prefix + \"_\" + the_item\n",
    "media_path = \"data/media\"\n",
    "\n",
    "print(data_path)\n",
    "print(media_path)\n",
    "\n",
    "try:\n",
    "    os.mkdir(\"data\")\n",
    "    print(\"made new data folder\")\n",
    "except:\n",
    "    pass\n",
    "\n",
    "try:\n",
    "    os.mkdir(\"data/media\")\n",
    "    print(\"made new media folder\")\n",
    "except:\n",
    "    pass\n",
    "\n",
    "try:\n",
    "    os.mkdir(data_path)\n",
    "    print(\"made new project folder\")\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found 407 saved posts.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    f = open(data_path+\"/metadata.json\",\"r\")\n",
    "    the_posts = json.load(f)\n",
    "    f.close()\n",
    "    print(\"found\",len(the_posts),\"saved posts.\")\n",
    "except:\n",
    "    the_posts = {}\n",
    "    print(\"This is a new scrape. No saved posts found.\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    f = open(\"config.json\",\"r\")\n",
    "except:\n",
    "    f = open(\"default-config.json\",\"r\")\n",
    "\n",
    "conf = json.load(f)\n",
    "f.close()\n",
    "if not the_item in conf:\n",
    "    conf[the_item] = conf[\"default\"]\n",
    "    f = open(\"config.json\",\"w\")\n",
    "    json.dump(conf,f)\n",
    "    f.close()\n",
    "    print(\"Updated the config file.\")\n",
    "deets = conf[the_item]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set up the web driver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if not os.getcwd() in os.get_exec_path():\n",
    "    print('adding path')\n",
    "    if platform.system() == \"Windows\":\n",
    "        os.environ[\"PATH\"] = os.environ[\"PATH\"] + \";\" + os.getcwd()\n",
    "    else:\n",
    "        os.environ[\"PATH\"] = os.environ[\"PATH\"] + \":\" + os.getcwd()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "driver.get(\"https://www.instagram.com/\"+insta_path+the_item+\"/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#catchoftheday • Instagram photos and videos\n"
     ]
    }
   ],
   "source": [
    "print(driver.title)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Send key strokes to scroll down the page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_8imhp\n"
     ]
    }
   ],
   "source": [
    "# first of all click on the \"Load more\" button\n",
    "\n",
    "# Instagram uses different class names for the \"Load more\" button, so it's necessary to check for more than one.\n",
    "load_more_codes = [\"_8imhp\",\"_oidfu\"]\n",
    "\n",
    "found_it = False\n",
    "\n",
    "for load_more_code in load_more_codes:\n",
    "    if not found_it:\n",
    "        try:\n",
    "            some_element = driver.find_element_by_class_name(load_more_code)\n",
    "            some_element.click()\n",
    "            found_it = True\n",
    "            print(load_more_code)\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "if not found_it:\n",
    "    print(\"there is only one page\")\n",
    "    some_element = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45 | overlap: 10\n"
     ]
    }
   ],
   "source": [
    "# then press space over an over again for a while\n",
    "last_how_many = -1\n",
    "how_many = 0\n",
    "overlap = set()\n",
    "all_links = []\n",
    "\n",
    "# why nine? there are 9 recurring top posts before you get into the recent posts\n",
    "while last_how_many<how_many and how_many<max_scrape and len(overlap)<=9:\n",
    "    \n",
    "    # send a key stroke to the page to scroll down\n",
    "    some_element = driver.find_element_by_tag_name(\"body\")\n",
    "    last_how_many = how_many\n",
    "    for i in range(3):\n",
    "        some_element.send_keys(\" \")\n",
    "        sleep(0.5)\n",
    "        \n",
    "    # check if the posts on the page have already been scraped\n",
    "    all_links = [u.get_attribute(\"href\").split(\"/\")[4] for u in driver.find_elements_by_class_name(\"_8mlbc\")]\n",
    "    overlap = set(the_posts.keys()).intersection(set(all_links))\n",
    "\n",
    "    how_many = len(driver.find_elements_by_class_name(\"_icyx7\"))\n",
    "    print(how_many,end=\" | \",flush=True)\n",
    "print(\"overlap:\",len(overlap))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45\n"
     ]
    }
   ],
   "source": [
    "# print the number of links/codes found in the page and save the list in a temporary file in case everything breaks down.\n",
    "print(len(all_links))\n",
    "f = open(data_path+\"/temp_links.json\",\"w\")\n",
    "json.dump(all_links,f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# scrape post pages + download media objects (photos or videos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def fixit(ss):\n",
    "    return \"\".join([s for s in ss if s in ascii_letters+digits]).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def clstr(ss):\n",
    "    return \"\".join([s for s in ss if s in printable])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def checkup(a_list,ss):\n",
    "    found_it = False\n",
    "    for a in a_list:\n",
    "        if a in ss:\n",
    "            found_it = True\n",
    "    return found_it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def extract_hashtags(ss):\n",
    "    ss = ss.replace(\"\\n\",\" \")\n",
    "    plist = ss.split(\" \")\n",
    "    tags = list(p for p in plist if len(p)>0 and p[0]==\"#\")\n",
    "    tt = []\n",
    "    for t in tags:\n",
    "        tt += t.split(\"#\")\n",
    "    return list(set(list(fixit(t) for t in tt if len(t)>0)))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def download_object(code):\n",
    "    if not os.path.isfile(media_path+\"/\"+the_posts[code].get(\"the_fname\",\"nofile.ppp\")):\n",
    "        try:\n",
    "            a_media_object = requests.get(the_posts[code][\"object_url\"],timeout=2)\n",
    "            with open(media_path+\"/\"+the_posts[code][\"the_fname\"], \"wb\") as f:\n",
    "                f.write(a_media_object.content)\n",
    "            print(\".\",end=\"\",flush=True)\n",
    "        except:\n",
    "            print(\"x\",end=\"\",flush=True)\n",
    "    else:\n",
    "        print(\"_\",end=\"\",flush=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# scrape new post pages + download media objects (photos or videos)\n",
    "def scrape_single_code(a_code,sleeper=1):\n",
    "    from ast import literal_eval\n",
    "    good_stuff = None\n",
    "    all_fine = True\n",
    "    try:\n",
    "        tt = requests.get(\"https://www.instagram.com/p/\"+a_code,timeout=2)\n",
    "        tt_soup = BeautifulSoup(tt.text, \"lxml\")\n",
    "        sleeper = 1\n",
    "    except:\n",
    "        all_fine = False\n",
    "        print(\"$\",end=\"\",flush=True)\n",
    "        sleep(sleeper)\n",
    "    \n",
    "    if all_fine:\n",
    "        # find all the javascript tags in the code\n",
    "        all_texts = [u.get_text() for u in tt_soup.find_all(\"script\")]\n",
    "\n",
    "        # extract the one that starts with this particular string\n",
    "        all_texts = [at for at in all_texts if \"window._sharedData\" in at[:50]][0]\n",
    "\n",
    "        # fix the JavaScript syntax so that it can be transformed to Python code\n",
    "        all_texts = all_texts.replace(\"window._sharedData\",\"\")\n",
    "        all_texts = all_texts.replace(\"true\",\"True\")\n",
    "        all_texts = all_texts.replace(\"false\",\"False\")\n",
    "        all_texts = all_texts.replace(\"null\",\"None\")\n",
    "        \n",
    "\n",
    "        # extract the dict from the code and reach for the good stuff\n",
    "        stuff = literal_eval(all_texts[3:-1])\n",
    "        try:\n",
    "            good_stuff = copy(stuff[\"entry_data\"][\"PostPage\"][0][\"media\"])\n",
    "        except:\n",
    "            return None\n",
    "        \n",
    "        # add some extra to the dict\n",
    "        good_stuff[\"scrape_ts\"] = str(datetime.now())[:19]\n",
    "        good_stuff[\"caption_tags\"] = extract_hashtags(good_stuff.get(\"caption\",\"\"))\n",
    "        \n",
    "        \n",
    "        if good_stuff[\"is_video\"]:\n",
    "            good_stuff[\"object_url\"] = good_stuff[\"video_url\"]\n",
    "            good_stuff[\"the_fname\"] = a_code + \".mp4\"\n",
    "        else:\n",
    "            good_stuff[\"object_url\"] = good_stuff[\"display_src\"]\n",
    "            good_stuff[\"the_fname\"] = a_code + \".jpg\"\n",
    "\n",
    "        print(\".\",end=\"\",flush=True)\n",
    "    return good_stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# scrape post pages for many codes\n",
    "def scrape_codes(some_codes):\n",
    "    new_stuff = {}\n",
    "    sleeptime=1\n",
    "    cc = 0\n",
    "    for a_code in some_codes:\n",
    "        cc += 1\n",
    "        if cc % 50==0: print(len(new_stuff))\n",
    "        new_stuff[a_code] = copy(scrape_single_code(a_code,sleeptime))\n",
    "        if new_stuff[a_code]==None:\n",
    "            del new_stuff[a_code]\n",
    "            sleeptime += sleeptime\n",
    "        else:\n",
    "            sleeptime = 1\n",
    "    return copy(new_stuff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# add the new stuff to the posts storage\n",
    "def update_post_storage(new_stuff):\n",
    "    cc = 0\n",
    "    for code in new_stuff:\n",
    "        cc += 1\n",
    "        if cc % 50 == 0: print()\n",
    "        if not code in the_posts:\n",
    "            the_posts[code] = copy(new_stuff[code])\n",
    "        if not \"activity\" in the_posts[code]:\n",
    "            the_posts[code][\"activity\"] = []\n",
    "        \n",
    "        tss = [ts[\"scrape_ts\"] for ts in the_posts[code][\"activity\"]]\n",
    "    \n",
    "        if not new_stuff[code][\"scrape_ts\"] in tss:\n",
    "            the_posts[code][\"activity\"] += [{\"scrape_ts\":new_stuff[code][\"scrape_ts\"],\n",
    "                                         \"comments\":new_stuff[code][\"comments\"],\n",
    "                                         \"likes\":new_stuff[code][\"likes\"],\n",
    "                                         \"video_views\":new_stuff[code].get(\"video_views\",\"not_a_video\")}]\n",
    "            print(len(the_posts[code][\"activity\"]),end=\"\",flush=True)\n",
    "        else:\n",
    "            # update everything but keep the activity record intact\n",
    "#            for u in new_stuff[code]:\n",
    "#                if u != 'activity':\n",
    "#                    the_posts[code][u] = new_stuff[code][u]\n",
    "            print(\"-\",end=\"\",flush=True)\n",
    "    print(\"\\nPosts in storage:\",len(the_posts))        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "...................................\n",
      "35\n",
      "11111111111111111111111111111111111\n",
      "Posts in storage: 405\n"
     ]
    }
   ],
   "source": [
    "# scrape new post pages (not the photo itself) and add to storage\n",
    "ns = scrape_codes([u for u in all_links if not u in the_posts])\n",
    "print(\"\\n\"+str(len(ns)))\n",
    "update_post_storage(ns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Do you want to update 1 posts? [Y/N] Y\n",
      "\n",
      "Posts in storage: 405\n"
     ]
    }
   ],
   "source": [
    "# update metadata for posts in storage\n",
    "\n",
    "update_frequency = 24 # hours\n",
    "\n",
    "codes_ready_for_update = []\n",
    "for code in the_posts:\n",
    "    latest_scrape = the_posts[code][\"activity\"][-1][\"scrape_ts\"]\n",
    "    time_since_latest_scrape = (datetime.now()-datetime.strptime(latest_scrape,\"%Y-%m-%d %H:%M:%S\")).total_seconds()\n",
    "    if time_since_latest_scrape > update_frequency * 3600:\n",
    "        codes_ready_for_update += [code]\n",
    "\n",
    "if len(codes_ready_for_update)>0 and input(\"Do you want to update \"+str(len(codes_ready_for_update))+\" posts? [Y/N] \") in \"Yy\":\n",
    "    ns = scrape_codes(codes_ready_for_update)\n",
    "    update_post_storage(ns)\n",
    "else:\n",
    "    print(\"No posts to update.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# exctract comments and tags in comments for posts in the storage. Add as two attributes in the_posts.\n",
    "# Also update caption tags from caption\n",
    "for code in the_posts:\n",
    "    \n",
    "    the_posts[code][\"caption_tags\"] = extract_hashtags(the_posts[code].get(\"caption\",\"\"))\n",
    "    \n",
    "    comments = []\n",
    "    for a in the_posts[code][\"activity\"]:\n",
    "        for c in a[\"comments\"][\"nodes\"]:\n",
    "            comments += [c['text']]\n",
    "    comments = list(set(comments))\n",
    "    the_posts[code][\"comm_texts\"]=comments\n",
    "    the_posts[code][\"comm_tags\"]=extract_hashtags(\" \".join(comments))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_.____________________.___.________.__.______.___\n",
      "__.._.____._________._.__________.________________\n",
      "_____.__._.______.________________________________\n",
      "_____________________._.._________.__.____________\n",
      "___._____________________________.____________.___\n",
      "______________________._______________.___________\n",
      "._______.__________.______________________________\n",
      "_________.___________.____________________.__._.__\n",
      "______"
     ]
    }
   ],
   "source": [
    "# download missing media objects\n",
    "cc = 0\n",
    "for code in the_posts:\n",
    "    cc += 1\n",
    "    if cc % 50 == 0: print()\n",
    "    download_object(code)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dump"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "f = open(data_path+\"/metadata.json\",\"w\")\n",
    "json.dump(the_posts,f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "driver.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Aggregate tags and labels into dataframes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The cells generate the following dataframes\n",
    "\n",
    "s_comm: all comments tags in all posts  \n",
    "s_cap: all caption tags in all posts  \n",
    "s_combined: all tags in all posts  \n",
    "s_labels: all labels (Google) in all posts  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# initialise \n",
    "all_labels = []\n",
    "comm_tags = []\n",
    "cap_tags = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for code in the_posts:\n",
    "    cap_tags += the_posts[code].get(\"caption_tags\",[])\n",
    "    comm_tags += the_posts[code].get(\"comm_tags\",[]) #include tags found in comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique comment tags: 667\n",
      "Unique caption tags: 2560\n",
      "Tags both in comments and captions: 232\n"
     ]
    }
   ],
   "source": [
    "print(\"Unique comment tags:\",len(set(comm_tags)))\n",
    "print(\"Unique caption tags:\",len(set(cap_tags)))\n",
    "print(\"Tags both in comments and captions:\",len(set(comm_tags).intersection(set(cap_tags))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s_comm = pd.DataFrame(Counter(comm_tags).most_common(100)[1:],columns=[\"tags\",\"freq\"])\n",
    "s_comm[\"type\"]=\"comment\"\n",
    "s_cap = pd.DataFrame(Counter(cap_tags).most_common(100)[1:],columns=[\"tags\",\"freq\"])\n",
    "s_cap[\"type\"]=\"caption\"\n",
    "s_combined = s_comm.append(s_cap)\n",
    "s_combined.index = range(0,len(s_combined))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for code in the_posts:\n",
    "    if \"labels\" in the_posts[code]:\n",
    "        all_labels += list(map(lambda x:x[\"description\"], the_posts[code][\"labels\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s_labels = pd.DataFrame(Counter(all_labels).most_common(40),columns=[\"labels\",\"freq\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s_comm.to_csv(data_path+\"/s_comm.csv\")\n",
    "s_cap.to_csv(data_path+\"/s_cap.csv\")\n",
    "s_combined.to_csv(data_path+\"/s_combined.csv\")\n",
    "s_labels.to_csv(data_path+\"/s_labels.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create HTML for the Photo Grid "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_total_likes(p):\n",
    "    likes = 0\n",
    "    for u in p['activity']:\n",
    "        if u[\"video_views\"] == \"not_a_video\":\n",
    "            likes += u[\"likes\"][\"count\"]\n",
    "        else:\n",
    "            likes += u[\"video_views\"] \n",
    "    return likes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if deets[\"display\"][\"selected_labels\"]==[]:\n",
    "    selected_labels = set(all_labels)\n",
    "else:\n",
    "    selected_labels = set(deets[\"display\"][\"selected_labels\"])\n",
    "    \n",
    "banned_labels = set(deets[\"display\"][\"banned_labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if deets[\"display\"][\"selected_tags\"]==[]:\n",
    "    selected_tags = set(cap_tags+comm_tags)\n",
    "else:\n",
    "    selected_tags = set(deets[\"display\"][\"selected_tags\"])\n",
    "    \n",
    "banned_tags = set(deets[\"display\"][\"banned_tags\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(len(the_posts),\"posts in total\")\n",
    "the_code = ''\n",
    "\n",
    "cc = 1\n",
    "the_size = 150\n",
    "include_this = True\n",
    "\n",
    "\n",
    "for code in the_posts:\n",
    "    include_this = True\n",
    "    if cc > 50000: break\n",
    "        \n",
    "    ##############################################################################\n",
    "    ## Start of the section where photos/videos are selected to be included or not\n",
    "    if 'labels' in the_posts[code]:\n",
    "        photo_labels = set(map(lambda x:x[\"description\"], the_posts[code][\"labels\"]))\n",
    "    else:\n",
    "        photo_labels = set()\n",
    "    \n",
    "    if  not the_posts[code][\"is_video\"] and len(all_labels)>0:\n",
    "        if len(photo_labels.intersection(selected_labels))==0 or \\\n",
    "        len(photo_labels.intersection(banned_labels))>0: \n",
    "            include_this = False\n",
    "\n",
    "    photo_tags =  set(the_posts[code][\"caption_tags\"]+the_posts[code][\"caption_tags\"])\n",
    "    if len(cap_tags+comm_tags)>0:\n",
    "        if len(photo_tags.intersection(selected_tags))==0 or \\\n",
    "        len(photo_tags.intersection(banned_tags))>0:\n",
    "            include_this = False\n",
    "\n",
    "    if get_total_likes(the_posts[code])<deets[\"display\"][\"min_likes\"] or \\\n",
    "    get_total_likes(the_posts[code])>deets[\"display\"][\"max_likes\"]:\n",
    "        include_this = False\n",
    "    \n",
    "    if the_posts[code][\"is_ad\"]: include_this = deets[\"display\"][\"include_ads\"]==\"yes\" and include_this\n",
    "    \n",
    "    if deets[\"display\"][\"include_videos\"]==\"none\" and the_posts[code][\"is_video\"]: include_this = False\n",
    "    if deets[\"display\"][\"include_photos\"]==\"none\" and not the_posts[code][\"is_video\"]: include_this = False    \n",
    "    \n",
    "    # the conditions below can turn back a \"False\" generated in the previous statements\n",
    "    if len(photo_labels)==0 and not the_posts[code][\"is_video\"]: include_this = True\n",
    "    if len(photo_tags)==0: include_this = True\n",
    "    \n",
    "    if deets[\"display\"][\"include_videos\"]==\"all\" and the_posts[code][\"is_video\"]: include_this = True\n",
    "    if deets[\"display\"][\"include_photos\"]==\"all\" and not the_posts[code][\"is_video\"]: include_this = True\n",
    "    ##################################################################################\n",
    "    ## End of the section for selecting photos/videos\n",
    "    ##################################################################################\n",
    "    \n",
    "    \n",
    "    if include_this:\n",
    "        cc += 1\n",
    "        if not the_posts[code][\"is_video\"]:\n",
    "        \n",
    "            the_code += '<div class=\"imgWrap\" width='+str(the_size)+' height='+str(the_size)+'>'\n",
    "            the_code += '<a href=\"https://www.instagram.com/p/'+code+'\" target=\"_blank\">'\n",
    "            the_code += '<img class=\"centered-and-cropped\" src=\"../media/'+the_posts[code]['the_fname']\n",
    "            the_code += '\" width='+str(the_size)+' height='+str(the_size)+' alt=\"\">'\n",
    "            the_code += '<p class=\"imgDescription\">'+clstr(the_posts[code].get(\"caption\",\"\")[:300])+'</p>'\n",
    "            the_code += '</a>'\n",
    "            the_code += '</div>&nbsp;'\n",
    "        \n",
    "        else:\n",
    "        \n",
    "            the_code += '<a href=\"https://www.instagram.com/p/'+code+'\" target=\"_blank\">'\n",
    "            the_code += '<video height=\"'+str(the_size)\n",
    "            the_code += '\" onmouseover=\"this.play();\" onmouseout=\"this.pause();\">'\n",
    "            the_code += '<source src=\"../media/'+the_posts[code]['the_fname']+'\" type=\"video/mp4\">'\n",
    "            the_code += '</video>&nbsp;'\n",
    "            the_code += '</a>'\n",
    "\n",
    "print(\"Ready to display\",cc-1,\"objects\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "f = open(data_path+\"/\"+the_item+\"_grid.html\",\"w\")\n",
    "f.write('<html><head><link rel=\"stylesheet\" href=\"../insta.css\"></head><body>')\n",
    "f.write(the_code)\n",
    "f.write(\"</body></html>\")\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}