Crypto Scrape.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Main Dependencies\n",
    "import pandas as pd\n",
    "import requests\n",
    "from tkinter import *\n",
    "from splinter import Browser\n",
    "from bs4 import BeautifulSoup\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "import time\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - Current google-chrome version is 88.0.4324\n",
      "[WDM] - Get LATEST driver version for 88.0.4324\n",
      "[WDM] - Get LATEST driver version for 88.0.4324\n",
      "[WDM] - Trying to download new driver from http://chromedriver.storage.googleapis.com/88.0.4324.96/chromedriver_win32.zip\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - Driver has been saved in cache [C:\\Users\\redea\\.wdm\\drivers\\chromedriver\\win32\\88.0.4324.96]\n"
     ]
    }
   ],
   "source": [
    "#Setup splinter\n",
    "executable_path = {'executable_path': ChromeDriverManager().install()}\n",
    "browser = Browser('chrome', **executable_path, headless=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Establish URL\n",
    "url = 'https://coinmarketcap.com/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = []\n",
    "\n",
    "for i in range(0, 43, 1):\n",
    "    loop_url = f'https://coinmarketcap.com/?page={i}'\n",
    "    browser.visit(loop_url)\n",
    "    \n",
    "    for j in range(10):\n",
    "        browser.execute_script(\"window.scrollBy(0,1000);\")\n",
    "        time.sleep(.5)\n",
    "        \n",
    "    html = browser.html\n",
    "    \n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    \n",
    "    table = soup.find('table', class_='cmc-table')\n",
    "    trs = table.find_all('tr')\n",
    "\n",
    "    for tr in trs:\n",
    "\n",
    "        token = {}\n",
    "\n",
    "        tds = tr.find_all('td')\n",
    "\n",
    "        try:\n",
    "            if tds[2].p is not None:\n",
    "                coin_name = tds[2].p.text\n",
    "                coin_code = tds[2].div.div.div.p.text\n",
    "            else:\n",
    "                coin_name = tds[2].find_all('span')[1].text\n",
    "                coin_code = tds[2].find_all('span')[2].text\n",
    "\n",
    "            coin_price = float(tds[3].text.replace(',','').strip('$'))\n",
    "            coin_24h = float(tds[4].text.strip('%'))\n",
    "            coin_7d = float(tds[5].text.strip('%'))\n",
    "            \n",
    "            try:\n",
    "                coin_market_cap = float(tds[6].text.replace(',','').strip('$'))\n",
    "            except Exception as e:\n",
    "                coin_market_cap = None\n",
    "                print(f'Warning: row [{i}] is missing market cap.')                \n",
    "\n",
    "            # from Kick: use this as an example to isolate volume, which is sometimes blank\n",
    "            try:\n",
    "                coin_volume_usd = float(tds[7].find_all('p')[0].text.replace(',','').strip('$'))\n",
    "                coin_volume_crypto_list = tds[7].find_all('p')[1].text.replace(',','').strip('$').split()\n",
    "                coin_volume = float(coin_volume_crypto_list[0])\n",
    "                coin_volume_token = coin_volume_crypto_list[1]\n",
    "            except Exception as e:\n",
    "                i = trs.index(tr)\n",
    "                print(f'Warning: row [{i}] is missing volume.')\n",
    "                coin_volume_usd = None\n",
    "                coin_volume_crypto_list = None\n",
    "                coin_volume = None\n",
    "                coin_volume_token = None\n",
    "\n",
    "            try:\n",
    "                coin_circulating_supply_list = tds[8].text.replace(',','').strip('$').split()\n",
    "                coin_circulating_supply = float(coin_circulating_supply_list[0])\n",
    "                coin_circulating_supply_token = coin_circulating_supply_list[1]\n",
    "            except Exception as e:\n",
    "                print(f'Warning: row [{i}] is missing circulating supply.')                \n",
    "                coin_circulating_supply_list = None\n",
    "                coin_circulating_supply = None\n",
    "                coin_circulating_supply_token = None\n",
    "                \n",
    "\n",
    "            # set up dictionary record\n",
    "            token['token'] = coin_code\n",
    "            token['name'] = coin_name\n",
    "            token['price'] = coin_price\n",
    "            token['24h'] = coin_24h\n",
    "            token['7d'] = coin_7d\n",
    "            token['market_cap'] = coin_market_cap\n",
    "            token['volume_usd'] = coin_volume_usd\n",
    "            token['volume'] = coin_volume\n",
    "            token['circulating_supply'] = coin_circulating_supply\n",
    "\n",
    "            # add to the list of dictionaries\n",
    "            tokens.append(token)\n",
    "\n",
    "        except Exception as e:\n",
    "            i = trs.index(tr)\n",
    "            #print(f'Error with index [{i}] of the tr.')\n",
    "            continue\n",
    "            \n",
    "browser.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create DataFrame\n",
    "token = pd.DataFrame(tokens)\n",
    "token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Extract Master array of NAMES ONLY for all Cryptos\n",
    "coins = token['name'].values\n",
    "#coins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Extract Master array of NAMES ONLY for all Cryptos\n",
    "codes = token['token'].values\n",
    "#codes\n",
    "#len(codes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Main Currency Query<h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Setting Up Drop Down Function\n",
    "root = Tk()\n",
    "root.geometry( \"200x200\" )\n",
    "def show():\n",
    "    label.config( text = clicked.get() )\n",
    "    \n",
    "options = coins\n",
    "clicked = StringVar()\n",
    "clicked.set('Click Me!')\n",
    "\n",
    "drop = OptionMenu( root, clicked, *options)\n",
    "drop.pack()\n",
    "\n",
    "button = Button(root, text = 'Select This Coin?', command =show).pack()\n",
    "\n",
    "label= Label(root, text = \" \")\n",
    "label.pack()\n",
    "\n",
    "root.mainloop()\n",
    "\n",
    "content = clicked.get()\n",
    "#content\n",
    "crypto = content\n",
    "crypto\n",
    "\n",
    "#Don't Forget to 'x' out of the prompt!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "code_filter = token[token['name']==crypto]\n",
    "code_df = code_filter.values[0]\n",
    "tick = code_df[0]\n",
    "tick"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#FORMAT QUERY\n",
    "lowercase = str.lower(crypto)\n",
    "formed = lowercase.replace(\" \",\"-\")\n",
    "#formed\n",
    "exchange = f'https://www.coindesk.com/price/{formed}'\n",
    "#exchange"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Yahoo Historic Search DB</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#You can sub the second number in the range for len(codes)\n",
    "history = []\n",
    "for c in range(0,11,1):\n",
    "    url = f'https://finance.yahoo.com/quote/{codes[c]}-USD/history?p={codes[c]}-USD'\n",
    "    hist = requests.get(url)\n",
    "    ho = BeautifulSoup(hist.text, 'html.parser')\n",
    "    try:\n",
    "        tabbed = pd.read_html(url)[0]\n",
    "    except:\n",
    "        continue\n",
    "    tabbed['Coin'] = codes[c]\n",
    "    history.append(tabbed)\n",
    "\n",
    "    \n",
    "yahoo = pd.concat(history)\n",
    "#yahoo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Export to CSV\n",
    "yahoo.to_csv('History.csv', index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Export to JSON\n",
    "import csv\n",
    "import json\n",
    "\n",
    "def make_json(csvFilePath, jsonFilePath):\n",
    "\n",
    "    data = {}\n",
    "     \n",
    "    with open(csvFilePath, encoding='utf-8') as csvf:\n",
    "        csvReader = csv.DictReader(csvf)\n",
    "         \n",
    "        for rows in csvReader:\n",
    "\n",
    "            key = rows['Coin']\n",
    "            data[key] = rows\n",
    "\n",
    "    with open(jsonFilePath, 'w', encoding='utf-8') as jsonf:\n",
    "        jsonf.write(json.dumps(data, indent=4))\n",
    "         \n",
    "csvFilePath = r'History.csv'\n",
    "jsonFilePath = r'History.json'\n",
    " \n",
    "make_json(csvFilePath, jsonFilePath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>API History Pull - Single Search</h1>\n",
    "\n",
    "<href>https://nomics.com/docs/#section/Introduction</href>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "API = '40925fa28c8e33ae59c34f12791f07d6'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = input('What date would you like to start at (yyyy-mm-dd)? ')\n",
    "end = input('What date would you like to end at (yyyy-mm-dd)? ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "volhist_url = f'https://api.nomics.com/v1/volume/history?key={API}&exchange={formed}&start={start}T00%3A00%3A00Z&end={end}T00%3A00%3A00Z'\n",
    "marketcap_url = f'https://api.nomics.com/v1/market-cap/history?key={API}&start={start}T00%3A00%3A00Z&end={end}T00%3A00%3A00Z'\n",
    "exchangerate_url = f'https://api.nomics.com/v1/exchange-rates/history?key={API}&currency={tick}&start={start}T00%3A00%3A00Z&end={end}T00%3A00%3A00Z'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Volume History\n",
    "response = requests.get(volhist_url)\n",
    "data = response.json()\n",
    "print(json.dumps(data, indent=4, sort_keys=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Market Cap\n",
    "response = requests.get(marketcap_url)\n",
    "data = response.json()\n",
    "print(json.dumps(data, indent=4, sort_keys=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exchange Rate\n",
    "response = requests.get(exchangerate_url)\n",
    "data = response.json()\n",
    "print(json.dumps(data, indent=4, sort_keys=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Exchange Information</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#NEW URL\n",
    "stats = f'https://coinmarketcap.com/currencies/{formed}/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cooking in the Kitchen\n",
    "analysis = requests.get(stats)\n",
    "broth = BeautifulSoup(analysis.text, 'html.parser')\n",
    "#broth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pulling Tables in HTML\n",
    "tables = broth.findAll('table')\n",
    "#tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#First Table\n",
    "first_table_scrape = tables[0]\n",
    "# first_table_scrape\n",
    "first_table_tds = first_table_scrape.find_all('td')\n",
    "#first_table_tds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Second Table\n",
    "second_table_scrape = tables[1]\n",
    "#second_table_scrape\n",
    "second_table_tds = second_table_scrape.find_all('td')\n",
    "#second_table_tds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Third Table\n",
    "third_table_scrape = tables[2]\n",
    "#third_table_scrape\n",
    "third_table_tds = third_table_scrape.find_all('td')\n",
    "#third_table_tds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fourth Table\n",
    "fourth_table_scrape = tables[3]\n",
    "#fourth_table_scrape\n",
    "fourth_table_tds = fourth_table_scrape.find_all('td')\n",
    "#fourth_table_tds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fifth Table\n",
    "fifth_table_scrape = tables[4]\n",
    "#fifth_table_scrape\n",
    "fifth_table_tds = fifth_table_scrape.find_all('td')\n",
    "#fifth_table_tds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pull ABOUT Narrative\n",
    "texts = broth.select('div.about___1OuKY')\n",
    "#texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Token Pull\n",
    "code = broth.findAll('div', class_='sc-16r8icm-0 fCTazK nameHeader___27HU_')[0].find_all('small')[0].getText()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Establishing List\n",
    "tables = []\n",
    "\n",
    "#Creating the Dictionary\n",
    "table = {}\n",
    "\n",
    "#First Table\n",
    "price = first_table_tds[0].get_text()\n",
    "pricechange24hr = first_table_tds[1].find_all('span')[0].get_text()\n",
    "percentchange24hr = first_table_tds[1].find_all('span')[1].get_text()\n",
    "low24hr = first_table_tds[2].find_all('div')[0].get_text().split(' /')[0]\n",
    "high24hr = first_table_tds[2].find_all('div')[1].get_text().split(' /')[0]\n",
    "tradingvolume = first_table_tds[3].find_all('span')[0].get_text()\n",
    "tradingvolumepercdelta = first_table_tds[3].find_all('span')[1].get_text()\n",
    "marketdominance = first_table_tds[4].find_all('span')[0].get_text()\n",
    "marketrank = first_table_tds[5].get_text().split('#')[1]\n",
    "\n",
    "#Second Table\n",
    "marketcap = second_table_tds[0].find_all('span')[0].get_text()\n",
    "fullydiluted = second_table_tds[1].find_all('span')[0].get_text()\n",
    "\n",
    "#Third Table\n",
    "yestlow = third_table_tds[0].find_all('div')[0].get_text().split(' /')[0]\n",
    "yesthigh = third_table_tds[0].find_all('div')[1].get_text().split(' /')[0]\n",
    "yestopen = third_table_tds[1].find_all('div')[0].get_text().split(' /')[0]\n",
    "yestclose = third_table_tds[1].find_all('div')[1].get_text().split(' /')[0]\n",
    "yestvol = third_table_tds[3].get_text()\n",
    "\n",
    "#Fourth Table\n",
    "d7low = fourth_table_tds[0].find_all('div')[0].get_text().split(' /')[0]\n",
    "d7high = fourth_table_tds[0].find_all('div')[1].get_text().split(' /')[0]\n",
    "d30low = fourth_table_tds[1].find_all('div')[0].get_text().split(' /')[0]\n",
    "d30high = fourth_table_tds[1].find_all('div')[1].get_text().split(' /')[0]\n",
    "d90low = fourth_table_tds[2].find_all('div')[0].get_text().split(' /')[0]\n",
    "d90high = fourth_table_tds[2].find_all('div')[1].get_text().split(' /')[0]\n",
    "wk52low = fourth_table_tds[3].find_all('div')[0].get_text().split(' /')[0]\n",
    "wk52high = fourth_table_tds[3].find_all('div')[1].get_text().split(' /')[0]\n",
    "alltimehigh = fourth_table_tds[4].find_all('span')[0].get_text()\n",
    "alltimehighperc = fourth_table_tds[4].find_all('span')[1].get_text()\n",
    "alltimelow = fourth_table_tds[5].find_all('span')[0].get_text()\n",
    "alltimelowperc = fourth_table_tds[5].find_all('span')[1].get_text()\n",
    "roi = fourth_table_tds[6].findAll('p')[0].getText()\n",
    "\n",
    "#Fifth Table\n",
    "circsupp = fifth_table_tds[0].get_text()\n",
    "totsupp = fifth_table_tds[1].get_text()\n",
    "mxsupp = fifth_table_tds[2].get_text()\n",
    "\n",
    "#Narrative\n",
    "intro1 = texts[1].find_all('p')[0].get_text()\n",
    "intro2 = texts[1].find_all('p')[1].get_text()\n",
    "fullintro = f'{intro1}{intro2}'\n",
    "\n",
    "#Building the Dictionary\n",
    "table['Code'] = code\n",
    "table['Price'] = price\n",
    "table['24hr Price Change'] = pricechange24hr\n",
    "table['24hr % Change'] = percentchange24hr\n",
    "table['24hr Low'] = low24hr\n",
    "table['24hr High'] = high24hr\n",
    "table['24hr Trading Volume'] = tradingvolume\n",
    "table['24hr Trading Volume % Change'] = tradingvolumepercdelta\n",
    "table['Market Dominance'] = marketdominance\n",
    "table['Market Rank'] = marketrank\n",
    "table['Market Cap'] = marketcap\n",
    "table['Fully Diluted Market Cap'] = fullydiluted\n",
    "table['Yesterday Low'] = yestlow\n",
    "table['Yesterday High'] = yesthigh\n",
    "table['Yesterday Open'] = yestopen\n",
    "table['Yesterday Close'] = yestclose\n",
    "table['Yesterday Volume'] = yestvol\n",
    "table['7 Day Low'] = d7low\n",
    "table['7 Day High'] = d7high\n",
    "table['30 Day Low'] = d30low\n",
    "table['30 Day High'] = d30high\n",
    "table['90 Day Low'] = d90low\n",
    "table['90 Day High'] = d90high\n",
    "table['52 Week Low'] = wk52low\n",
    "table['52 Week High'] = wk52high\n",
    "table['All Time High'] = alltimehigh\n",
    "table['All Time High %'] = alltimehighperc\n",
    "table['All Time Low'] = alltimelow\n",
    "table['All Time Low %'] = alltimelowperc\n",
    "table['Return on Investment'] = roi\n",
    "table['Cirulating Supply'] = circsupp\n",
    "table['Total Supply'] = totsupp\n",
    "table['Max Supply'] = mxsupp\n",
    "table[\"About\"] = fullintro\n",
    "table['Link'] = stats\n",
    "\n",
    "#Appending\n",
    "tables.append(table)\n",
    "\n",
    "#Print Final Dictionary\n",
    "tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOGO PHOTO\n",
    "lc_code = str.lower(code)\n",
    "snap = f'https://cryptologos.cc/logos/{formed}-{lc_code}-logo.png?v=010'\n",
    "print(snap)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Market Purchase Information</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#NEW URL\n",
    "markets = f'https://coinmarketcap.com/currencies/{formed}/markets/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cooking in the Kitchen\n",
    "whereto = requests.get(markets)\n",
    "bisque = BeautifulSoup(whereto.text, 'html.parser')\n",
    "#bisque"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pulling Tables in HTML\n",
    "frames = bisque.findAll('table')\n",
    "board = frames[0]\n",
    "#board"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Setting up trs -- remember to skip [0] tr since it's a header\n",
    "board_trs = board.find_all('tr')\n",
    "#board_trs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Top 100 Crypto Markets for Query\n",
    "guides = []\n",
    "\n",
    "for i in range(1,101,1):\n",
    "    \n",
    "    tds = board_trs[i].find_all('td')\n",
    "\n",
    "    guide = {}\n",
    "\n",
    "    source = tds[1].find_all('p')[0].getText()\n",
    "    pairs = tds[2].find_all('a')[0].getText()\n",
    "    web = tds[2].find_all('a')[0].get('href')\n",
    "    price = tds[3].get_text()\n",
    "    volume = tds[4].find_all('p')[0].getText()\n",
    "    volumeperc = tds[5].find_all('div')[0].get_text()\n",
    "    liquid = tds[6].find_all('p')[0].getText()\n",
    "    confidence = tds[7].find_all('div')[0].get_text()\n",
    "    updated = tds[8].find_all('p')[0].getText()\n",
    "\n",
    "    guide['Source'] = source\n",
    "    guide['Currency'] = pairs\n",
    "    guide['URL'] = web\n",
    "    guide['Price'] = price\n",
    "    guide['Volume'] = volume\n",
    "    guide['Volume %'] = volumeperc\n",
    "    guide['Liquidity'] = liquid\n",
    "    guide['Confidence'] = confidence\n",
    "    guide['Last Updated'] = updated\n",
    "\n",
    "    guides.append(guide)\n",
    "\n",
    "#guides\n",
    "blackmarket = pd.DataFrame(guides)\n",
    "blackmarket"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Historical Data</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#NEW URL - Creating Date List\n",
    "dates = 'https://coinmarketcap.com/historical/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cooking in the Kitchen\n",
    "select = requests.get(dates)\n",
    "pho = BeautifulSoup(select.text, 'html.parser')\n",
    "#pho"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pull history of all dates\n",
    "links = pho.find_all('div',class_='cmc-bottom-margin-2x')[0].find_all('a',class_='historical-link cmc-link')\n",
    "#links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create DataFrame\n",
    "dates = []\n",
    "\n",
    "for l in range(36,len(links),1):\n",
    "    \n",
    "    date = {}\n",
    "    \n",
    "    nums=links[l].get('href').split(\"/\")[2]\n",
    "    \n",
    "    date['Raw'] = nums\n",
    "\n",
    "    dates.append(date)\n",
    "    \n",
    "timeline = pd.DataFrame(dates)\n",
    "#timeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reformat DataFrame\n",
    "timeline['Form'] = timeline['Raw'].astype(str)\n",
    "timeline['Day'] = timeline['Form'].str[6:8]\n",
    "timeline['Month'] = timeline['Form'].str[4:6]\n",
    "timeline['Year'] = timeline['Form'].str[0:4]\n",
    "timeline.drop('Form', axis=1, inplace=True)\n",
    "#timeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Add the Query column to the DataFrame\n",
    "timeline[\"Query\"] = timeline[\"Month\"] + \"/\" + timeline[\"Day\"] + \"/\" + timeline[\"Year\"]\n",
    "#timeline\n",
    "specs = timeline['Query'].values\n",
    "#specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create a Date select function\n",
    "root = Tk()\n",
    "root.geometry( \"200x200\" )\n",
    "def show():\n",
    "    label.config( text = clicked.get() )\n",
    "    \n",
    "options = specs\n",
    "clicked = StringVar()\n",
    "clicked.set('Click Me!')\n",
    "\n",
    "drop = OptionMenu( root, clicked, *options)\n",
    "drop.pack()\n",
    "\n",
    "button = Button(root, text = 'Select This Date?', command =show).pack()\n",
    "\n",
    "label= Label(root, text = \" \")\n",
    "label.pack()\n",
    "\n",
    "root.mainloop()\n",
    "\n",
    "content = clicked.get()\n",
    "#content\n",
    "birth = content\n",
    "birth\n",
    "\n",
    "#Don't Forget to 'x' out of the prompt!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reformat the User Input for URL variable\n",
    "reform = birth.split('/')\n",
    "history_input = f'{reform[2]}{reform[0]}{reform[1]}'\n",
    "#history_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#NEW URL\n",
    "history = f'https://coinmarketcap.com/historical/{history_input}/'\n",
    "#print(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cooking in the Kitchen\n",
    "story = requests.get(history)\n",
    "chowder = BeautifulSoup(story.text, 'html.parser')\n",
    "#chowder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dive into the table\n",
    "exhibits = chowder.findAll('tbody')\n",
    "itinerary = exhibits[0]\n",
    "#itinerary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Locate table data\n",
    "itinerary_trs = itinerary.find_all('tr')\n",
    "#itinerary_trs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Max of Top 200 Crypto Histories\n",
    "logs = []\n",
    "\n",
    "for w in range(0,len(itinerary),1):\n",
    "    \n",
    "    tds = itinerary_trs[w].find_all('td')\n",
    "\n",
    "    log = {}\n",
    "\n",
    "    rank = tds[0].find_all('div')[0].get_text()\n",
    "    name = tds[1].find_all('a')[0].getText()\n",
    "    symbol = tds[2].find_all('div')[0].get_text()\n",
    "    marketcap = tds[3].find_all('p')[0].getText()\n",
    "    price = tds[4].find_all('a')[0].getText()\n",
    "    circsupp = tds[5].find_all('div')[0].get_text()\n",
    "    vol24h = tds[6].find_all('a')[0].getText()\n",
    "    p1h = tds[7].find_all('div')[0].get_text()\n",
    "    p24h = tds[8].find_all('div')[0].get_text()\n",
    "    p7d = tds[9].find_all('div')[0].get_text()\n",
    "\n",
    "    log['Rank'] = rank\n",
    "    log['Name'] = name\n",
    "    log['Symbol'] = symbol\n",
    "    log['Market Cap'] = marketcap\n",
    "    log['Price'] = price\n",
    "    log['Circulating Supply'] = circsupp\n",
    "    log['1 Hour Percent Change'] = p1h\n",
    "    log['24 Hour Percent Change'] = p24h\n",
    "    log['7 Day Percent Change'] = p7d\n",
    "    \n",
    "\n",
    "    logs.append(log)\n",
    "\n",
    "#logs\n",
    "print(f'-------------Historical Snapshot for {formed} on {birth}-------------')\n",
    "textbook = pd.DataFrame(logs)\n",
    "textbook.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Tweepy</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SHOULD BE PULLED DAILY TO NOT OVERLOAD API REQUESTS\n",
    "!pip install tweepy\n",
    "import tweepy\n",
    "import json\n",
    "from config import Consumer_Key, Consumer_Secret, Access_Token, Access_Token_Secret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Establishing API - REMEMBER TO CHANGE SEARCH TERM\n",
    "auth = tweepy.OAuthHandler(Consumer_Key,Consumer_Secret)\n",
    "auth.set_access_token(Access_Token,Access_Token_Secret)\n",
    "api = tweepy.API(auth, parser=tweepy.parsers.JSONParser())\n",
    "#30 Day Analysis\n",
    "tokes = api.search_30_day('month',crypto)\n",
    "#tokes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Testing Pull\n",
    "results = tokes['results']\n",
    "#len(results)\n",
    "#results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#30 Day DataFrame\n",
    "tweets = []\n",
    "\n",
    "for r in results:\n",
    "    \n",
    "    tweet = {}\n",
    "    \n",
    "\n",
    "    dates = r['created_at'].split(\" \")\n",
    "    date = dates[0] + \"., \" + dates[1] + \". \" + dates[2]\n",
    "    text = r['text']\n",
    "    location = r['user']['location']\n",
    "    description = r['user']['description']\n",
    "    favs = r['user']['favourites_count']\n",
    "    screen = r['user']['screen_name']\n",
    "    \n",
    "    tweet['Screen Name'] = f'@{screen}'\n",
    "    tweet['Date'] = date\n",
    "    tweet['Text'] = text\n",
    "    tweet['Location'] = location\n",
    "    tweet['Description'] = description\n",
    "    tweet['Favorited'] = favs\n",
    "    \n",
    "    tweets.append(tweet)\n",
    "\n",
    "#tweets\n",
    "bird = pd.DataFrame(tweets)\n",
    "bird.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Full Search\n",
    "full = api.search(crypto)\n",
    "#full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Testing Pull\n",
    "series = full['statuses']\n",
    "#len(series)\n",
    "#series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Full Search DataFrame\n",
    "steps = []\n",
    "\n",
    "for s in series:\n",
    "    \n",
    "    step = {}\n",
    "    \n",
    "\n",
    "    dates = s['created_at'].split(\" \")\n",
    "    date = dates[0] + \"., \" + dates[1] + \". \" + dates[2]\n",
    "    text = s['text']\n",
    "    location = s['user']['location']\n",
    "    description = s['user']['description']\n",
    "    favs = s['user']['favourites_count']\n",
    "    screen = s['user']['screen_name']\n",
    "    \n",
    "    step['Screen Name'] = f'@{screen}'\n",
    "    step['Date'] = date\n",
    "    step['Text'] = text\n",
    "    step['Location'] = location\n",
    "    step['Description'] = description\n",
    "    step['Favorited'] = favs\n",
    "    \n",
    "    steps.append(step)\n",
    "\n",
    "#steps\n",
    "wing = pd.DataFrame(steps)\n",
    "wing.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Reddit</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Establish Python Reddit Wrapper\n",
    "!pip install praw\n",
    "import praw\n",
    "from config import Client_ID, Client_Secret, User_Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Connect Credentials\n",
    "reddit = praw.Reddit(client_id=Client_ID, client_secret=Client_Secret, user_agent=User_Agent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pull Top 50 Posts for Query SubReddit\n",
    "posts = []\n",
    "try:\n",
    "    crypt_read = reddit.subreddit(formed)\n",
    "except Exception as e:\n",
    "    print('Sorry! That Crypto does not have a SubReddit. Redirecting you to the general CrytoCurrency SubReddit.')\n",
    "    crypt_read = reddit.subreddit('cryptocurrency')\n",
    "    \n",
    "for post in crypt_read.hot(limit=50):\n",
    "    posts.append([post.title, post.score, post.subreddit, post.url, post.num_comments])\n",
    "    karma = pd.DataFrame(posts,columns=['Title','Score','SubReddit','URL','# of Comments'])\n",
    "        \n",
    "karma.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>General Media - Top News Stories</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "media = 'https://www.coindesk.com/category/markets'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "journal = requests.get(media)\n",
    "eggdrop = BeautifulSoup(journal.text, 'html.parser')\n",
    "#eggdrop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dumpling = eggdrop.find('div',class_='page-area-dotted').find('section',class_='page-area-dotted-content').find('section',class_='list-body')\n",
    "#dumpling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Top 9 News Stories from CoinDesk -- MAYBE PUT THIS IN A TICKER FOR VISUAL?\n",
    "articles = []\n",
    "\n",
    "for a in range(0,len(dumpling),1):\n",
    "\n",
    "    article = {}\n",
    "\n",
    "    title = dumpling.find_all('div',class_='text-content')[a].find_all('h4')[0].getText()\n",
    "    link = dumpling.find_all('div',class_='text-content')[a].find_all('a')[1].get('href')\n",
    "    description = dumpling.find_all('div',class_='text-content')[a].find_all('p')[0].getText()\n",
    "\n",
    "    article['Title'] = title\n",
    "    article['Link'] = f'https://www.coindesk.com{link}'\n",
    "    article['Description'] = description\n",
    "\n",
    "    articles.append(article)\n",
    "\n",
    "#articles\n",
    "newspaper = pd.DataFrame(articles)\n",
    "newspaper"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>101 Guides to CrytoCurrency (CoinDesk)</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classroom = 'https://www.coindesk.com/learn'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "desk = requests.get(classroom)\n",
    "chickenoodle = BeautifulSoup(desk.text, 'html.parser')\n",
    "#chickenoodle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "noodle = chickenoodle.find('div',class_='page-area-dotted').find('section',class_='page-area-dotted-content').find('section',class_='list-body')\n",
    "#noodle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#101 Guides - Count of 50\n",
    "books = []\n",
    "\n",
    "for b in range(0,len(noodle),1):\n",
    "\n",
    "    book = {}\n",
    "\n",
    "    title = noodle.find_all('div',class_='text-content')[b].find_all('h4')[0].getText()\n",
    "    link = noodle.find_all('div',class_='text-content')[b].find_all('a')[1].get('href')\n",
    "    description = noodle.find_all('div',class_='text-content')[b].find_all('p')[0].getText()\n",
    "\n",
    "    book['Title'] = title\n",
    "    book['Link'] = f'https://www.coindesk.com{link}'\n",
    "    book['Description'] = description\n",
    "\n",
    "    books.append(book)\n",
    "\n",
    "#books\n",
    "library = pd.DataFrame(books)\n",
    "library.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Cryto Legality Around the Globe</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dora = 'https://en.wikipedia.org/wiki/Legality_of_bitcoin_by_country_or_territory'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "backpack = requests.get(dora)\n",
    "ramen = BeautifulSoup(backpack.text, 'html.parser')\n",
    "#ramen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pulling Tables in HTML\n",
    "bowls = ramen.findAll('table')\n",
    "#bowls\n",
    "#len(bowls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "      <th>Legal Status</th>\n",
       "      <th>Blurb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Algeria</td>\n",
       "      <td>Illegal</td>\n",
       "      <td>According to the \"Journal Officiel\" (28 Decemb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Egypt</td>\n",
       "      <td>Illegal</td>\n",
       "      <td>\"Egypt’s Dar al-Ifta, the primary Islamic legi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Morocco</td>\n",
       "      <td>Illegal</td>\n",
       "      <td>On 20 November 2017 the exchange office issued...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Legal</td>\n",
       "      <td>As of 17 January 2017, The Central Bank of Nig...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Mauritius</td>\n",
       "      <td>Legal</td>\n",
       "      <td>The Financial Services Commission of Mauritius...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>Luxembourg</td>\n",
       "      <td>Legal</td>\n",
       "      <td>The Commission de Surveillance du Secteur Fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>Netherlands</td>\n",
       "      <td>Legal</td>\n",
       "      <td>As of 2017[update], virtual currencies such as...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>Legal</td>\n",
       "      <td>As of 2017[update], the government of the Unit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>Australia</td>\n",
       "      <td>Legal</td>\n",
       "      <td>In December 2013, the governor of the Reserve ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>New Zealand</td>\n",
       "      <td>Legal</td>\n",
       "      <td>The Reserve Bank of New Zealand states: \"Non-b...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>85 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Country Legal Status  \\\n",
       "0          Algeria      Illegal   \n",
       "1            Egypt      Illegal   \n",
       "2          Morocco      Illegal   \n",
       "3          Nigeria        Legal   \n",
       "4        Mauritius        Legal   \n",
       "..             ...          ...   \n",
       "80      Luxembourg        Legal   \n",
       "81     Netherlands        Legal   \n",
       "82  United Kingdom        Legal   \n",
       "83       Australia        Legal   \n",
       "84     New Zealand        Legal   \n",
       "\n",
       "                                                Blurb  \n",
       "0   According to the \"Journal Officiel\" (28 Decemb...  \n",
       "1   \"Egypt’s Dar al-Ifta, the primary Islamic legi...  \n",
       "2   On 20 November 2017 the exchange office issued...  \n",
       "3   As of 17 January 2017, The Central Bank of Nig...  \n",
       "4   The Financial Services Commission of Mauritius...  \n",
       "..                                                ...  \n",
       "80  The Commission de Surveillance du Secteur Fina...  \n",
       "81  As of 2017[update], virtual currencies such as...  \n",
       "82  As of 2017[update], the government of the Unit...  \n",
       "83  In December 2013, the governor of the Reserve ...  \n",
       "84  The Reserve Bank of New Zealand states: \"Non-b...  \n",
       "\n",
       "[85 rows x 3 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Establishing List\n",
    "rules = []\n",
    "\n",
    "for bowl in range(3,len(bowls),1):\n",
    "    \n",
    "    head = bowls[bowl].find_all('tr')\n",
    "    \n",
    "    for x in range(1,len(head),1):\n",
    "\n",
    "        #Creating the Dictionary\n",
    "        rule = {}\n",
    "        \n",
    "        try:\n",
    "            \n",
    "            title = head[x].find_all('td')[0].find_all('a')[0].getText()\n",
    "            legal = head[x].find_all('td')[1].get_text().split('\\n')[0].split(' ')[1]\n",
    "            body = head[x].find_all('td')[1].find_all('p')[0].getText().split('\\n')[0]\n",
    "        \n",
    "        except:\n",
    "            continue\n",
    "\n",
    "        #Building the Dictionary\n",
    "        rule['Country'] = title\n",
    "        rule['Legal Status'] = legal\n",
    "        rule['Blurb'] = body\n",
    "\n",
    "        #Appending\n",
    "        rules.append(rule)\n",
    "\n",
    "#Print Final Data\n",
    "#rules\n",
    "atlantis = pd.DataFrame(rules)\n",
    "atlantis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "atlantis.to_csv(\"Countries.csv\", index=False, header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>BitCoin Depot</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import API\n",
    "from config import Google_API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bee = 'https://bitcoindepot.com/locations/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reading URL\n",
    "gold = requests.get(bee)\n",
    "mapo = BeautifulSoup(gold.text, 'html.parser')\n",
    "#mapo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pulling Addresses in HTML\n",
    "tofu = mapo.findAll('div',class_='list-country-card')\n",
    "#tofu\n",
    "#len(tofu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Establishing List\n",
    "atms = []\n",
    "\n",
    "for t in range(0,42,1):\n",
    "    \n",
    "    addresses = tofu[t].findAll('span',class_='list-country-list-text')\n",
    "    states = tofu[t].findAll('a',class_='get-direction-link')\n",
    "    hours = tofu[t].findAll('span',class_='list-country-list-time')\n",
    "\n",
    "    for a in range(0,len(addresses),1):\n",
    "        \n",
    "        atm = {}\n",
    "        \n",
    "        ave = addresses[a].getText().split(', ')[2].split(' ')\n",
    "        s = ' '\n",
    "        street = s.join(ave)\n",
    "        name = addresses[a].getText().split(', ')[1]\n",
    "    \n",
    "        for s in range(0,len(states),1):\n",
    "\n",
    "            atm = {}\n",
    "        \n",
    "            raw = states[s].get('href').split('+')\n",
    "            pull = len(raw)\n",
    "            state = raw[pull-2]\n",
    "            zipcode = raw[pull-1]\n",
    "            city = raw[pull-3]\n",
    "\n",
    "        for h in range(0,len(hours),1):\n",
    "\n",
    "            hour = hours[h].getText().replace('\\r\\n',', ')\n",
    "            \n",
    "            atm['Name'] = name\n",
    "            atm['Hours'] = hour\n",
    "            atm['Address'] = f'{street} {city}, {state}, {zipcode}'\n",
    "            atm['Street'] = street.split(',')[0]\n",
    "            atm['City'] = city\n",
    "            atm['State'] = state\n",
    "            atm['Zip Code'] = zipcode\n",
    "\n",
    "        atms.append(atm)\n",
    "#atms\n",
    "bitcoin = pd.DataFrame(atms)\n",
    "bitcoin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Adding Columns\n",
    "bitcoin['Latitude'] = ''\n",
    "bitcoin['Longitude'] = ''\n",
    "bitcoin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Geocode -- this takes a long time to run\n",
    "params = {'key': Google_API}\n",
    "for index, row in bitcoin.iterrows():\n",
    "    base_url = 'https://maps.googleapis.com/maps/api/geocode/json'\n",
    "    \n",
    "    street = row['Street']\n",
    "    city = row['City']\n",
    "    state = row['State']\n",
    "    zipcode = row['Zip Code']\n",
    "    \n",
    "    params['address'] = f'{street},{city},{state},{zipcode}'\n",
    "    \n",
    "    depots_lat_lng = requests.get(base_url, params=params)\n",
    "    depots_lat_lng = depots_lat_lng.json()\n",
    "    \n",
    "    try:\n",
    "        bitcoin.loc[index, 'Latitude'] = depots_lat_lng['results'][0]['geometry']['location']['lat']\n",
    "        bitcoin.loc[index, 'Longitude'] = depots_lat_lng['results'][0]['geometry']['location']['lng']\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Finalize All Data\n",
    "bitcoin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Export as CSV\n",
    "bitcoin.to_csv(\"ATMs.csv\", index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Export as JSON\n",
    "import csv\n",
    "import json\n",
    "\n",
    "def make_json(csvFilePath, jsonFilePath):\n",
    "\n",
    "    data = {}\n",
    "     \n",
    "    with open(csvFilePath, encoding='utf-8') as csvf:\n",
    "        csvReader = csv.DictReader(csvf)\n",
    "         \n",
    "        for rows in csvReader:\n",
    "\n",
    "            key = rows['Name']\n",
    "            data[key] = rows\n",
    "\n",
    "    with open(jsonFilePath, 'w', encoding='utf-8') as jsonf:\n",
    "        jsonf.write(json.dumps(data, indent=4))\n",
    "         \n",
    "csvFilePath = r'ATMs.csv'\n",
    "jsonFilePath = r'ATMs.json'\n",
    " \n",
    "make_json(csvFilePath, jsonFilePath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}