Crypto Scrape-DEVTEST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Grabbing all Cryptos</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Main Dependencies\n",
    "import pandas as pd\n",
    "import requests\n",
    "from tkinter import *\n",
    "from splinter import Browser\n",
    "from bs4 import BeautifulSoup\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "import time\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Setup splinter\n",
    "executable_path = {'executable_path': ChromeDriverManager().install()}\n",
    "browser = Browser('chrome', **executable_path, headless=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Establish URL\n",
    "url = 'https://coinmarketcap.com/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = []\n",
    "\n",
    "for i in range(0, 43, 1):\n",
    "    loop_url = f'https://coinmarketcap.com/?page={i}'\n",
    "    browser.visit(loop_url)\n",
    "    \n",
    "    for j in range(10):\n",
    "        browser.execute_script(\"window.scrollBy(0,1000);\")\n",
    "        time.sleep(.5)\n",
    "        \n",
    "    html = browser.html\n",
    "    \n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    \n",
    "    table = soup.find('table', class_='cmc-table')\n",
    "    trs = table.find_all('tr')\n",
    "\n",
    "    for tr in trs:\n",
    "\n",
    "        token = {}\n",
    "\n",
    "        tds = tr.find_all('td')\n",
    "\n",
    "        try:\n",
    "            if tds[2].p is not None:\n",
    "                coin_name = tds[2].p.text\n",
    "                coin_code = tds[2].div.div.div.p.text\n",
    "            else:\n",
    "                coin_name = tds[2].find_all('span')[1].text\n",
    "                coin_code = tds[2].find_all('span')[2].text\n",
    "\n",
    "            coin_price = float(tds[3].text.replace(',','').strip('$'))\n",
    "            coin_24h = float(tds[4].text.strip('%'))\n",
    "            coin_7d = float(tds[5].text.strip('%'))\n",
    "            \n",
    "            try:\n",
    "                coin_market_cap = float(tds[6].text.replace(',','').strip('$'))\n",
    "            except Exception as e:\n",
    "                coin_market_cap = None\n",
    "                print(f'Warning: row [{i}] is missing market cap.')                \n",
    "\n",
    "            # from Kick: use this as an example to isolate volume, which is sometimes blank\n",
    "            try:\n",
    "                coin_volume_usd = float(tds[7].find_all('p')[0].text.replace(',','').strip('$'))\n",
    "                coin_volume_crypto_list = tds[7].find_all('p')[1].text.replace(',','').strip('$').split()\n",
    "                coin_volume = float(coin_volume_crypto_list[0])\n",
    "                coin_volume_token = coin_volume_crypto_list[1]\n",
    "            except Exception as e:\n",
    "                i = trs.index(tr)\n",
    "                print(f'Warning: row [{i}] is missing volume.')\n",
    "                coin_volume_usd = None\n",
    "                coin_volume_crypto_list = None\n",
    "                coin_volume = None\n",
    "                coin_volume_token = None\n",
    "\n",
    "            try:\n",
    "                coin_circulating_supply_list = tds[8].text.replace(',','').strip('$').split()\n",
    "                coin_circulating_supply = float(coin_circulating_supply_list[0])\n",
    "                coin_circulating_supply_token = coin_circulating_supply_list[1]\n",
    "            except Exception as e:\n",
    "                print(f'Warning: row [{i}] is missing circulating supply.')                \n",
    "                coin_circulating_supply_list = None\n",
    "                coin_circulating_supply = None\n",
    "                coin_circulating_supply_token = None\n",
    "                \n",
    "\n",
    "            # set up dictionary record\n",
    "            token['token'] = coin_code\n",
    "            token['name'] = coin_name\n",
    "            token['price'] = coin_price\n",
    "            token['24h'] = coin_24h\n",
    "            token['7d'] = coin_7d\n",
    "            token['market_cap'] = coin_market_cap\n",
    "            token['volume_usd'] = coin_volume_usd\n",
    "            token['volume'] = coin_volume\n",
    "            token['circulating_supply'] = coin_circulating_supply\n",
    "\n",
    "            # add to the list of dictionaries\n",
    "            tokens.append(token)\n",
    "\n",
    "        except Exception as e:\n",
    "            i = trs.index(tr)\n",
    "            #print(f'Error with index [{i}] of the tr.')\n",
    "            continue\n",
    "            \n",
    "browser.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create DataFrame\n",
    "token = pd.DataFrame(tokens)\n",
    "#token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Extract Master array of NAMES ONLY for all Cryptos\n",
    "coins = token['name'].values\n",
    "#coins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Extract Master array of NAMES ONLY for all Cryptos\n",
    "codes = token['token'].values\n",
    "codes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Exchange Data - Database<h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Grab All URLS\n",
    "links = []\n",
    "for c in range(0,len(coins),1):\n",
    "    lowercase = codes[c].lower()\n",
    "    for r in ((\" \", \"-\"), (\".\", \"-\")):\n",
    "        lowercase = lowercase.replace(*r)\n",
    "        exchange = f'https://coinmarketcap.com/currencies/{lowercase}/'\n",
    "        \n",
    "    #Appending\n",
    "    links.append(exchange)\n",
    "\n",
    "#Print Final Dictionary\n",
    "#links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOGO PHOTO\n",
    "logos = []\n",
    "for c in range(0,len(coins),1):\n",
    "    lowercase = coins[c].lower()\n",
    "    lc_code = codes[c].lower()\n",
    "    for r in ((\" \", \"-\"), (\".\", \"-\")):\n",
    "        lowercase = lowercase.replace(*r)\n",
    "        snap = f'https://cryptologos.cc/logos/{lowercase}-{lc_code}-logo.png?v=010'\n",
    "        \n",
    "    #Appending\n",
    "    logos.append(snap)\n",
    "\n",
    "#Print Final Dictionary\n",
    "#logos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dollars = []\n",
    "\n",
    "for l in range(0,len(links),1):\n",
    "    \n",
    "    dollar = {}\n",
    "        \n",
    "    stats = links[l]\n",
    "    analysis = requests.get(stats)\n",
    "    broth = BeautifulSoup(analysis.text, 'html.parser')\n",
    "    tables = broth.findAll('table')\n",
    "    \n",
    "    try:\n",
    "        first_table_scrape = tables[0]\n",
    "        first_table_tds = first_table_scrape.find_all('td')\n",
    "    except:\n",
    "        continue\n",
    "    try:\n",
    "        second_table_scrape = tables[1]\n",
    "        second_table_tds = second_table_scrape.find_all('td')\n",
    "    except:\n",
    "        continue\n",
    "    try:\n",
    "        third_table_scrape = tables[2]\n",
    "        third_table_tds = third_table_scrape.find_all('td')\n",
    "    except:\n",
    "        continue\n",
    "    try:\n",
    "        fourth_table_scrape = tables[3]\n",
    "        fourth_table_tds = fourth_table_scrape.find_all('td')\n",
    "    except:\n",
    "        continue\n",
    "    try:\n",
    "        fifth_table_scrape = tables[4]\n",
    "        fifth_table_tds = fifth_table_scrape.find_all('td')\n",
    "    except:\n",
    "        continue\n",
    "\n",
    "    texts = broth.select('div.about___1OuKY')\n",
    "    code = broth.findAll('div', class_='sc-16r8icm-0 fCTazK nameHeader___27HU_')[0].find_all('small')[0].getText()\n",
    "        \n",
    "    try:\n",
    "        price = first_table_tds[0].get_text()\n",
    "    except: \n",
    "        price = None\n",
    "    try:\n",
    "        pricechange24hr = first_table_tds[1].find_all('span')[0].get_text()\n",
    "        percentchange24hr = first_table_tds[1].find_all('span')[1].get_text()\n",
    "    except:\n",
    "        pricechange24hr = None\n",
    "        percentchange24hr = None\n",
    "    try:\n",
    "        low24hr = first_table_tds[2].find_all('div')[0].get_text().split(' /')[0]\n",
    "        high24hr = first_table_tds[2].find_all('div')[1].get_text().split(' /')[0]\n",
    "    except:\n",
    "        low24hr = None\n",
    "        high24hr = None\n",
    "    try:\n",
    "        tradingvolume = first_table_tds[3].find_all('span')[0].get_text()\n",
    "        tradingvolumepercdelta = first_table_tds[3].find_all('span')[1].get_text()\n",
    "    except:\n",
    "        tradingvolume = None\n",
    "        tradingvolumepercdelta = None\n",
    "    try:\n",
    "        marketdominance = first_table_tds[4].find_all('span')[0].get_text()\n",
    "        marketrank = first_table_tds[5].get_text().split('#')[1]\n",
    "        marketcap = second_table_tds[0].find_all('span')[0].get_text()\n",
    "        fullydiluted = second_table_tds[1].find_all('span')[0].get_text()\n",
    "    except:\n",
    "        marketdominance = None\n",
    "        marketrank = None\n",
    "        marketcap = None\n",
    "        fullydiluted = None\n",
    "    \n",
    "    try:\n",
    "        yestlow = third_table_tds[0].find_all('div')[0].get_text().split(' /')[0]\n",
    "        yesthigh = third_table_tds[0].find_all('div')[1].get_text().split(' /')[0]\n",
    "        yestopen = third_table_tds[1].find_all('div')[0].get_text().split(' /')[0]\n",
    "        yestclose = third_table_tds[1].find_all('div')[1].get_text().split(' /')[0]\n",
    "        yestvol = third_table_tds[3].get_text()\n",
    "    except:\n",
    "        yestlow = None\n",
    "        yesthigh = None\n",
    "        yestopen = None\n",
    "        yestclose = None\n",
    "        yestvol = None\n",
    "\n",
    "    try:\n",
    "        d7low = fourth_table_tds[0].find_all('div')[0].get_text().split(' /')[0]\n",
    "        d7high = fourth_table_tds[0].find_all('div')[1].get_text().split(' /')[0]\n",
    "    except:\n",
    "        d7low = None\n",
    "        d7high = None\n",
    "    try:\n",
    "        d30low = fourth_table_tds[1].find_all('div')[0].get_text().split(' /')[0]\n",
    "        d30high = fourth_table_tds[1].find_all('div')[1].get_text().split(' /')[0]  \n",
    "    except:\n",
    "        d30low = None\n",
    "        d30high = None\n",
    "    try:\n",
    "        d90low = fourth_table_tds[2].find_all('div')[0].get_text().split(' /')[0]\n",
    "        d90high = fourth_table_tds[2].find_all('div')[1].get_text().split(' /')[0] \n",
    "    except:\n",
    "        d90low = None\n",
    "        d90high = None\n",
    "    try:\n",
    "        wk52low = fourth_table_tds[3].find_all('div')[0].get_text().split(' /')[0]\n",
    "        wk52high = fourth_table_tds[3].find_all('div')[1].get_text().split(' /')[0]\n",
    "    except:\n",
    "        wk52low = None\n",
    "        wk52high = None\n",
    "    try:\n",
    "        alltimehigh = fourth_table_tds[4].find_all('span')[0].get_text()\n",
    "        alltimehighperc = fourth_table_tds[4].find_all('span')[1].get_text()\n",
    "        alltimelow = fourth_table_tds[5].find_all('span')[0].get_text()\n",
    "        alltimelowperc = fourth_table_tds[5].find_all('span')[1].get_text()\n",
    "    except:\n",
    "        alltimehigh = None\n",
    "        alltimehighperc = None\n",
    "        alltimelow = None\n",
    "        alltimelowperc = None\n",
    "    try:\n",
    "        roi = fourth_table_tds[6].findAll('p')[0].getText()\n",
    "    except:\n",
    "        roi = None\n",
    "    try:\n",
    "        circsupp = fifth_table_tds[0].get_text()\n",
    "        totsupp = fifth_table_tds[1].get_text()\n",
    "        mxsupp = fifth_table_tds[2].get_text()\n",
    "    except:\n",
    "        circsupp = None\n",
    "        totsupp = None\n",
    "        mxsupp = None\n",
    "    intro1 = texts[1].find_all('p')[0].get_text()\n",
    "    intro2 = texts[1].find_all('p')[1].get_text()\n",
    "    fullintro = f'{intro1}{intro2}'\n",
    "\n",
    "    dollar['Code'] = code\n",
    "    dollar['Price'] = price\n",
    "    dollar['24hr Price Change'] = pricechange24hr\n",
    "    dollar['24hr % Change'] = percentchange24hr\n",
    "    dollar['24hr Low'] = low24hr\n",
    "    dollar['24hr High'] = high24hr\n",
    "    dollar['24hr Trading Volume'] = tradingvolume\n",
    "    dollar['24hr Trading Volume % Change'] = tradingvolumepercdelta\n",
    "    dollar['Market Dominance'] = marketdominance\n",
    "    dollar['Market Rank'] = marketrank\n",
    "    dollar['Market Cap'] = marketcap\n",
    "    dollar['Fully Diluted Market Cap'] = fullydiluted\n",
    "    dollar['Yesterday Low'] = yestlow\n",
    "    dollar['Yesterday High'] = yesthigh\n",
    "    dollar['Yesterday Open'] = yestopen\n",
    "    dollar['Yesterday Close'] = yestclose\n",
    "    dollar['Yesterday Volume'] = yestvol\n",
    "    dollar['7 Day Low'] = d7low\n",
    "    dollar['7 Day High'] = d7high\n",
    "    dollar['30 Day Low'] = d30low\n",
    "    dollar['30 Day High'] = d30high\n",
    "    dollar['90 Day Low'] = d90low\n",
    "    dollar['90 Day High'] = d90high\n",
    "    dollar['52 Week Low'] = wk52low\n",
    "    dollar['52 Week High'] = wk52high\n",
    "    dollar['All Time High'] = alltimehigh\n",
    "    dollar['All Time High %'] = alltimehighperc\n",
    "    dollar['All Time Low'] = alltimelow\n",
    "    dollar['All Time Low %'] = alltimelowperc\n",
    "    dollar['Return on Investment'] = roi\n",
    "    dollar['Cirulating Supply'] = circsupp\n",
    "    dollar['Total Supply'] = totsupp\n",
    "    dollar['Max Supply'] = mxsupp\n",
    "    dollar[\"About\"] = fullintro\n",
    "    dollar['Link'] = stats\n",
    "\n",
    "    #Appending\n",
    "    dollars.append(dollar)\n",
    "\n",
    "#Print Final Dictionary\n",
    "#dollars\n",
    "all_exchange = pd.DataFrame(dollars)\n",
    "all_exchange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Export to CSV\n",
    "all_exchange.to_csv('ExchangeInfo.csv', index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Export to JSON\n",
    "import csv\n",
    "import json\n",
    "\n",
    "def make_json(csvFilePath, jsonFilePath):\n",
    "\n",
    "    data = {}\n",
    "     \n",
    "    with open(csvFilePath, encoding='utf-8') as csvf:\n",
    "        csvReader = csv.DictReader(csvf)\n",
    "         \n",
    "        for rows in csvReader:\n",
    "\n",
    "            key = rows['Code']\n",
    "            data[key] = rows\n",
    "\n",
    "    with open(jsonFilePath, 'w', encoding='utf-8') as jsonf:\n",
    "        jsonf.write(json.dumps(data, indent=4))\n",
    "         \n",
    "csvFilePath = r'ExchangeInfo.csv'\n",
    "jsonFilePath = r'ExchangeInfo.json'\n",
    " \n",
    "make_json(csvFilePath, jsonFilePath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Historical Data - DataBase</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#NEW URL - Creating Date List\n",
    "dates = 'https://coinmarketcap.com/historical/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cooking in the Kitchen\n",
    "select = requests.get(dates)\n",
    "pho = BeautifulSoup(select.text, 'html.parser')\n",
    "#pho"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pull history of all dates\n",
    "links = pho.find_all('div',class_='cmc-bottom-margin-2x')[0].find_all('a',class_='historical-link cmc-link')\n",
    "#links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create DataFrame\n",
    "dates = []\n",
    "\n",
    "for l in range(36,len(links),1):\n",
    "    \n",
    "    date = {}\n",
    "    \n",
    "    nums=links[l].get('href').split(\"/\")[2]\n",
    "    \n",
    "    dates.append(nums)\n",
    "    \n",
    "#dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Grab All URLS\n",
    "links = []\n",
    "for d in range(0,len(dates),1):\n",
    "    history = f'https://coinmarketcap.com/historical/{dates[d]}/'\n",
    "        \n",
    "    #Appending\n",
    "    links.append(history)\n",
    "\n",
    "#Print Final Dictionary\n",
    "#links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logs = []\n",
    "\n",
    "for l in range(0,len(links),1):\n",
    "    story = requests.get(links[l])\n",
    "    chowder = BeautifulSoup(story.text, 'html.parser')\n",
    "    exhibits = chowder.findAll('tbody')\n",
    "    try:\n",
    "        itinerary = exhibits[0]\n",
    "    except:\n",
    "        print('Scraping Error. Data Not Available')\n",
    "    itinerary_trs = itinerary.find_all('tr')\n",
    "    for w in range(0,len(itinerary),1):\n",
    "\n",
    "        tds = itinerary_trs[w].find_all('td')\n",
    "\n",
    "        log = {}\n",
    "\n",
    "        rank = tds[0].find_all('div')[0].get_text()\n",
    "        name = tds[1].find_all('a')[0].getText()\n",
    "        symbol = tds[2].find_all('div')[0].get_text()\n",
    "        marketcap = tds[3].find_all('p')[0].getText()\n",
    "        price = tds[4].find_all('a')[0].getText()\n",
    "        circsupp = tds[5].find_all('div')[0].get_text()\n",
    "        vol24h = tds[6].find_all('a')[0].getText()\n",
    "        p1h = tds[7].find_all('div')[0].get_text()\n",
    "        p24h = tds[8].find_all('div')[0].get_text()\n",
    "        p7d = tds[9].find_all('div')[0].get_text()\n",
    "\n",
    "        log['Date'] = links[l].split('/')[4]\n",
    "        log['Rank'] = rank\n",
    "        log['Name'] = name\n",
    "        log['Symbol'] = symbol\n",
    "        log['Market Cap'] = marketcap\n",
    "        log['Price'] = price\n",
    "        log['Circulating Supply'] = circsupp\n",
    "        log['1 Hour Percent Change'] = p1h\n",
    "        log['24 Hour Percent Change'] = p24h\n",
    "        log['7 Day Percent Change'] = p7d\n",
    "\n",
    "\n",
    "        logs.append(log)\n",
    "\n",
    "#logs\n",
    "textbook = pd.DataFrame(logs)\n",
    "textbook['Date'] = pd.to_datetime(textbook['Date'], format='%Y%m%d',errors='coerce')\n",
    "#textbook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Export to CSV\n",
    "textbook.to_csv('History.csv', index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Export to JSON\n",
    "import csv\n",
    "import json\n",
    "\n",
    "def make_json(csvFilePath, jsonFilePath):\n",
    "\n",
    "    data = {}\n",
    "     \n",
    "    with open(csvFilePath, encoding='utf-8') as csvf:\n",
    "        csvReader = csv.DictReader(csvf)\n",
    "         \n",
    "        for rows in csvReader:\n",
    "\n",
    "            key = rows['Date']\n",
    "            data[key] = rows\n",
    "\n",
    "    with open(jsonFilePath, 'w', encoding='utf-8') as jsonf:\n",
    "        jsonf.write(json.dumps(data, indent=4))\n",
    "         \n",
    "csvFilePath = r'History.csv'\n",
    "jsonFilePath = r'History.json'\n",
    " \n",
    "make_json(csvFilePath, jsonFilePath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}