workbook August 2019.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from pprint import pprint\n",
    "import progressbar, csv, json, re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_cache_dir = Path('./__cache__/tweets/')\n",
    "user_cache_dir = Path('./__cache__/users/')\n",
    "\n",
    "def get_tweet(_id):\n",
    "    _json = None\n",
    "    cache_file = tweet_cache_dir / _id\n",
    "    if cache_file.is_file:\n",
    "        with cache_file.open() as f:\n",
    "            _json = json.load(fp=f)\n",
    "    return(_json)\n",
    "\n",
    "def expand_tags_tsv(file):\n",
    "    all_captions = {}\n",
    "    count_not_downloaded = 0\n",
    "    _json = None\n",
    "    \n",
    "    if Path(file).is_file(): _ = [Path(file)]\n",
    "    elif Path(file).is_dir(): _ = Path(file).glob(pattern=\"*.tsv\")\n",
    "    else:\n",
    "        raise RuntimeError(f\"Cannot interpret passed argument: {file}.\")\n",
    "    for file in _:\n",
    "        with Path(file).open(\"r\") as f:\n",
    "            _len = len(f.readlines())\n",
    "        with Path(file).open(\"r\") as f:\n",
    "            bar = progressbar.ProgressBar(max_value=_len).start()\n",
    "            reader = csv.DictReader(f, delimiter='\\t')\n",
    "            for i, rows in enumerate(reader):\n",
    "                bar.update(i)\n",
    "                skip = False\n",
    "                try:\n",
    "                    id_int = int(rows['id_str'])\n",
    "                    _json = get_tweet(rows['id_str'])\n",
    "                except Exception as e:\n",
    "                    # if \"invalid literal\" in str(e): print(f\"Not a valid ID on line {i}.\")\n",
    "                    skip = True\n",
    "                \n",
    "                if not skip and _json is not None:\n",
    "                    if _json['json_source'] == \"Twitter\":\n",
    "                        # We have a JSON file and it comes from Twitter\n",
    "                        \n",
    "                        if not _json['retweeted']:\n",
    "                            clean_text = _json['full_text']\n",
    "                            clean_text = ' '.join(re.sub(\"(^RT )|(#[A-Za-z0-9_]+)|(@[A-Za-z0-9_]+)|([^0-9A-Za-z \\t])|(\\w+:\\/\\/\\S+)|(\\b(I|II|III|IV|V|VI|VII|VIII|IX|X|XI|XII|XIII|XIV|XV|XVI|XVII|XVIII|XIX|XX))\",\" \",clean_text).split())\n",
    "                            clean_text = clean_text.lower()\n",
    "                            if \"burlesque\" in clean_text:\n",
    "                                clean_text_first_8_words = \" \".join(clean_text.split(\" \")[:8])\n",
    "                                if clean_text_first_8_words not in all_captions: all_captions[clean_text_first_8_words] = 0\n",
    "                                all_captions[clean_text_first_8_words] += 1\n",
    "                    elif _json['json_source'] == \"TAGS\":\n",
    "                        # We have a limited JSON file and it comes from TAGS (which also means it contains the error?)\n",
    "                        if \"63\" in _json['error']: pass # User has been suspended\n",
    "                        elif \"179\" in _json['error']: pass # Not authorized to see status.\n",
    "                        elif \"144\" in _json['error']: pass # No status found with that ID.\n",
    "                        elif \"34\" in _json['error']: pass # No such page exists\n",
    "                        else: \n",
    "                            pprint(_json['error'])\n",
    "                            print(tweet_cache_dir / str(id_int), \"\\n\")\n",
    "                        pass\n",
    "                elif _json is None:\n",
    "                    count_not_downloaded += 1\n",
    "    if count_not_downloaded > 0: print(f\"Warning: {count_not_downloaded} tweets could not be loaded (because they haven't been downloaded yet).\")\n",
    "    return(all_captions)\n",
    "                    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% (83991 of 84122) |################# | Elapsed Time: 0:01:06 ETA:   0:00:00"
     ]
    }
   ],
   "source": [
    "all_captions = expand_tags_tsv('../../datasets/tags-tsv/burlesque/TAGS - burlesque 1 - Archive.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('photography series captures male burlesque performers all around', 16),\n",
      " ('8 gennaio bianconiglio burlesque boylesque fe', 13),\n",
      " ('do count as maps here s one of', 6),\n",
      " ('male burlesque entertainers please wear pasties in solidarity', 4),\n",
      " ('all male burlesque show glitter bombs the world', 4),\n",
      " ('bella serata iniziamo il 2016 col botto mega', 4),\n",
      " ('sparkly beaded flat boylesque or burlesque pasties in', 3),\n",
      " ('naughty little cabaret chicago s only male burlesque', 3),\n",
      " ('don t miss the lovely and 24 other', 3),\n",
      " ('saturday january 23rd come see 25 burlesque and', 3),\n",
      " ('at 5 8pm wednesday feb 17 burlesque boylesque', 2),\n",
      " ('challenging gender beliefs of being a feminine heterosexual', 2),\n",
      " ('i vote for a burlesque and boylesque show', 2),\n",
      " ('when you lose your job and your wife', 2),\n",
      " ('variety of upcoming events at the encore entertainment', 2),\n",
      " ('burlesque and boylesque and drag queens oh my', 2),\n",
      " ('briefs fantastic all male burlesque show udderbelly festival', 2),\n",
      " ('saturday jan 23rd be here 25 amazing burlesque', 2),\n",
      " ('what is the name of chandler s father', 2),\n",
      " ('the official fifty shades of grey male burlesque', 2),\n",
      " ('a few looks from sinnersaintburlesque', 1),\n",
      " ('look out for our next post in the', 1),\n",
      " ('don t recall rr ever dancing in male', 1),\n",
      " ('all male revue on main stage at the', 1),\n",
      " ('the burlesque community is excited to have you', 1),\n",
      " ('free introductory boylesque class happening this week in', 1),\n",
      " ('male burlesque dancers wanted', 1),\n",
      " ('supporting my new boyfriend harderburlesque the', 1),\n",
      " ('taking a burlesque strip class tonight', 1),\n",
      " ('brilliant male burlesque safe to watch but a', 1),\n",
      " ('princes of the night male burlesque show 2', 1),\n",
      " ('missed at last night watch these boys burlesque', 1),\n",
      " ('off to llandudno for the north wales burlesque', 1),\n",
      " ('i m on a show at blush tonight', 1),\n",
      " ('madonna shooting the sex book at lq outtake', 1),\n",
      " ('ross what s the name of chandler s', 1),\n",
      " ('your burlesque or boylesque name is first pet', 1),\n",
      " ('calling all males did you know that male', 1),\n",
      " ('royal purple halter bra ribbon tie neck detail', 1),\n",
      " ('i have found my male burlesque name at', 1),\n",
      " ('i found him by accident in 2014 at', 1),\n",
      " ('replay sexy male burlesque', 1),\n",
      " ('live on sexy male burlesque', 1),\n",
      " ('really what is this programming male burlesque from', 1)]\n"
     ]
    }
   ],
   "source": [
    "import operator\n",
    "sorted_all_captions = sorted(all_captions.items(), key=operator.itemgetter(1), reverse=True)\n",
    "for i, caption in enumerate(sorted_all_captions):\n",
    "    if int(caption[1]) >= 2: pass\n",
    "    else: del sorted_all_captions[i]\n",
    "pprint(sorted_all_captions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}