abs_main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "download audible books details and store them in a more useful format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import json\n",
    "import abs_reader\n",
    "import abs_download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "working folder is c:\\Users\\Willi\\OneDrive\\Repos\\Audible-book-scraper\n"
     ]
    }
   ],
   "source": [
    "''' setup folders and files'''\n",
    "if not 'workbookDir' in globals():          # get the local working directory\n",
    "    workbookDir = os.getcwd()               # needed to work with local html files\n",
    "print('working folder is',workbookDir)\n",
    "download_folder = workbookDir + '\\\\downloads\\\\'\n",
    "csvfile = workbookDir + '\\\\books.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sff1.html already exists, sff2.html already exists, sff3.html already exists, sff4.html already exists, mt1.html already exists, mt2.html already exists, mt3.html already exists, mt4.html already exists, mt5.html already exists, mt6.html already exists, "
     ]
    }
   ],
   "source": [
    "''' download html files '''\n",
    "with open('urls.json') as r:\n",
    "    pages_to_download = json.load(r)\n",
    "abs_download.download_html(pages_to_download,download_folder,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "csv already exists\n"
     ]
    }
   ],
   "source": [
    "'''process html files to csv'''\n",
    "abs_reader.process_folder(download_folder,csvfile,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_ed3bf_\" ><thead>    <tr>        <th class=\"col_heading level0 col0\" >name</th>        <th class=\"col_heading level0 col1\" >author</th>        <th class=\"col_heading level0 col2\" >stars</th>        <th class=\"col_heading level0 col3\" >ratings</th>        <th class=\"col_heading level0 col4\" >url</th>        <th class=\"col_heading level0 col5\" >full url</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                                <td id=\"T_ed3bf_row0_col0\" class=\"data row0 col0\" >Sword of Destiny</td>\n",
       "                        <td id=\"T_ed3bf_row0_col1\" class=\"data row0 col1\" >Andrzej Sapkowski</td>\n",
       "                        <td id=\"T_ed3bf_row0_col2\" class=\"data row0 col2\" >5.0</td>\n",
       "                        <td id=\"T_ed3bf_row0_col3\" class=\"data row0 col3\" >18322</td>\n",
       "                        <td id=\"T_ed3bf_row0_col4\" class=\"data row0 col4\" >B017WP4D6C</td>\n",
       "                        <td id=\"T_ed3bf_row0_col5\" class=\"data row0 col5\" >www.audible.com/pd/B017WP4D6C</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_ed3bf_row1_col0\" class=\"data row1 col0\" >Into the Fire</td>\n",
       "                        <td id=\"T_ed3bf_row1_col1\" class=\"data row1 col1\" >Gregg Hurwitz</td>\n",
       "                        <td id=\"T_ed3bf_row1_col2\" class=\"data row1 col2\" >5.0</td>\n",
       "                        <td id=\"T_ed3bf_row1_col3\" class=\"data row1 col3\" >6639</td>\n",
       "                        <td id=\"T_ed3bf_row1_col4\" class=\"data row1 col4\" >1799751511</td>\n",
       "                        <td id=\"T_ed3bf_row1_col5\" class=\"data row1 col5\" >www.audible.com/pd/1799751511</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_ed3bf_row2_col0\" class=\"data row2 col0\" >Nevermore</td>\n",
       "                        <td id=\"T_ed3bf_row2_col1\" class=\"data row2 col1\" >Dannika Dark</td>\n",
       "                        <td id=\"T_ed3bf_row2_col2\" class=\"data row2 col2\" >5.0</td>\n",
       "                        <td id=\"T_ed3bf_row2_col3\" class=\"data row2 col3\" >2341</td>\n",
       "                        <td id=\"T_ed3bf_row2_col4\" class=\"data row2 col4\" >B07MMWS5TR</td>\n",
       "                        <td id=\"T_ed3bf_row2_col5\" class=\"data row2 col5\" >www.audible.com/pd/B07MMWS5TR</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_ed3bf_row3_col0\" class=\"data row3 col0\" >Monster's Mercy</td>\n",
       "                        <td id=\"T_ed3bf_row3_col1\" class=\"data row3 col1\" >William D. Arand</td>\n",
       "                        <td id=\"T_ed3bf_row3_col2\" class=\"data row3 col2\" >5.0</td>\n",
       "                        <td id=\"T_ed3bf_row3_col3\" class=\"data row3 col3\" >2068</td>\n",
       "                        <td id=\"T_ed3bf_row3_col4\" class=\"data row3 col4\" >B087YRMKKB</td>\n",
       "                        <td id=\"T_ed3bf_row3_col5\" class=\"data row3 col5\" >www.audible.com/pd/B087YRMKKB</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_ed3bf_row4_col0\" class=\"data row4 col0\" >Undeading Bells</td>\n",
       "                        <td id=\"T_ed3bf_row4_col1\" class=\"data row4 col1\" >Drew Hayes</td>\n",
       "                        <td id=\"T_ed3bf_row4_col2\" class=\"data row4 col2\" >5.0</td>\n",
       "                        <td id=\"T_ed3bf_row4_col3\" class=\"data row4 col3\" >1934</td>\n",
       "                        <td id=\"T_ed3bf_row4_col4\" class=\"data row4 col4\" >1977346987</td>\n",
       "                        <td id=\"T_ed3bf_row4_col5\" class=\"data row4 col5\" >www.audible.com/pd/1977346987</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x2cd248336d0>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''' display books \n",
    "    5 star only         '''\n",
    "df_books = pd.read_csv(csvfile)\n",
    "df_books.sort_values('ratings',ascending=False).loc[df_books['stars'] == 5].head(5).style.hide_index().format({'stars': \"{:.2}\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_d9d4b_\" ><thead>    <tr>        <th class=\"col_heading level0 col0\" >name</th>        <th class=\"col_heading level0 col1\" >author</th>        <th class=\"col_heading level0 col2\" >stars</th>        <th class=\"col_heading level0 col3\" >ratings</th>        <th class=\"col_heading level0 col4\" >url</th>        <th class=\"col_heading level0 col5\" >full url</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                                <td id=\"T_d9d4b_row0_col0\" class=\"data row0 col0\" >The Last Wish</td>\n",
       "                        <td id=\"T_d9d4b_row0_col1\" class=\"data row0 col1\" >Andrzej Sapkowski</td>\n",
       "                        <td id=\"T_d9d4b_row0_col2\" class=\"data row0 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row0_col3\" class=\"data row0 col3\" >37150</td>\n",
       "                        <td id=\"T_d9d4b_row0_col4\" class=\"data row0 col4\" >B00WX7Y0W2</td>\n",
       "                        <td id=\"T_d9d4b_row0_col5\" class=\"data row0 col5\" >www.audible.com/pd/B00WX7Y0W2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row1_col0\" class=\"data row1 col0\" >Steelheart</td>\n",
       "                        <td id=\"T_d9d4b_row1_col1\" class=\"data row1 col1\" >Brandon Sanderson</td>\n",
       "                        <td id=\"T_d9d4b_row1_col2\" class=\"data row1 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row1_col3\" class=\"data row1 col3\" >35984</td>\n",
       "                        <td id=\"T_d9d4b_row1_col4\" class=\"data row1 col4\" >B00ECDZ08I</td>\n",
       "                        <td id=\"T_d9d4b_row1_col5\" class=\"data row1 col5\" >www.audible.com/pd/B00ECDZ08I</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row2_col0\" class=\"data row2 col0\" >Children of Time</td>\n",
       "                        <td id=\"T_d9d4b_row2_col1\" class=\"data row2 col1\" >Adrian Tchaikovsky</td>\n",
       "                        <td id=\"T_d9d4b_row2_col2\" class=\"data row2 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row2_col3\" class=\"data row2 col3\" >30996</td>\n",
       "                        <td id=\"T_d9d4b_row2_col4\" class=\"data row2 col4\" >B071Y9TTHC</td>\n",
       "                        <td id=\"T_d9d4b_row2_col5\" class=\"data row2 col5\" >www.audible.com/pd/B071Y9TTHC</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row3_col0\" class=\"data row3 col0\" >Leviathan Wakes</td>\n",
       "                        <td id=\"T_d9d4b_row3_col1\" class=\"data row3 col1\" >James S. A. Corey</td>\n",
       "                        <td id=\"T_d9d4b_row3_col2\" class=\"data row3 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row3_col3\" class=\"data row3 col3\" >23521</td>\n",
       "                        <td id=\"T_d9d4b_row3_col4\" class=\"data row3 col4\" >B073H9PF2D</td>\n",
       "                        <td id=\"T_d9d4b_row3_col5\" class=\"data row3 col5\" >www.audible.com/pd/B073H9PF2D</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row4_col0\" class=\"data row4 col0\" >The Secrets She Keeps</td>\n",
       "                        <td id=\"T_d9d4b_row4_col1\" class=\"data row4 col1\" >Michael Robotham</td>\n",
       "                        <td id=\"T_d9d4b_row4_col2\" class=\"data row4 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row4_col3\" class=\"data row4 col3\" >19585</td>\n",
       "                        <td id=\"T_d9d4b_row4_col4\" class=\"data row4 col4\" >B0716VL73Z</td>\n",
       "                        <td id=\"T_d9d4b_row4_col5\" class=\"data row4 col5\" >www.audible.com/pd/B0716VL73Z</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row5_col0\" class=\"data row5 col0\" >Monster Hunter International</td>\n",
       "                        <td id=\"T_d9d4b_row5_col1\" class=\"data row5 col1\" >Larry Correia</td>\n",
       "                        <td id=\"T_d9d4b_row5_col2\" class=\"data row5 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row5_col3\" class=\"data row5 col3\" >19301</td>\n",
       "                        <td id=\"T_d9d4b_row5_col4\" class=\"data row5 col4\" >B004RGQ6UQ</td>\n",
       "                        <td id=\"T_d9d4b_row5_col5\" class=\"data row5 col5\" >www.audible.com/pd/B004RGQ6UQ</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row6_col0\" class=\"data row6 col0\" >Watchers</td>\n",
       "                        <td id=\"T_d9d4b_row6_col1\" class=\"data row6 col1\" >Dean Koontz</td>\n",
       "                        <td id=\"T_d9d4b_row6_col2\" class=\"data row6 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row6_col3\" class=\"data row6 col3\" >18626</td>\n",
       "                        <td id=\"T_d9d4b_row6_col4\" class=\"data row6 col4\" >B07D2JG8TH</td>\n",
       "                        <td id=\"T_d9d4b_row6_col5\" class=\"data row6 col5\" >www.audible.com/pd/B07D2JG8TH</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row7_col0\" class=\"data row7 col0\" >Agent to the Stars</td>\n",
       "                        <td id=\"T_d9d4b_row7_col1\" class=\"data row7 col1\" >John Scalzi</td>\n",
       "                        <td id=\"T_d9d4b_row7_col2\" class=\"data row7 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row7_col3\" class=\"data row7 col3\" >14681</td>\n",
       "                        <td id=\"T_d9d4b_row7_col4\" class=\"data row7 col4\" >B004DR2CNE</td>\n",
       "                        <td id=\"T_d9d4b_row7_col5\" class=\"data row7 col5\" >www.audible.com/pd/B004DR2CNE</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row8_col0\" class=\"data row8 col0\" >The Poet</td>\n",
       "                        <td id=\"T_d9d4b_row8_col1\" class=\"data row8 col1\" >Michael Connelly</td>\n",
       "                        <td id=\"T_d9d4b_row8_col2\" class=\"data row8 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row8_col3\" class=\"data row8 col3\" >14357</td>\n",
       "                        <td id=\"T_d9d4b_row8_col4\" class=\"data row8 col4\" >B002UZN6SQ</td>\n",
       "                        <td id=\"T_d9d4b_row8_col5\" class=\"data row8 col5\" >www.audible.com/pd/B002UZN6SQ</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row9_col0\" class=\"data row9 col0\" >Still Life</td>\n",
       "                        <td id=\"T_d9d4b_row9_col1\" class=\"data row9 col1\" >Louise Penny</td>\n",
       "                        <td id=\"T_d9d4b_row9_col2\" class=\"data row9 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row9_col3\" class=\"data row9 col3\" >12187</td>\n",
       "                        <td id=\"T_d9d4b_row9_col4\" class=\"data row9 col4\" >B00JS9K7OI</td>\n",
       "                        <td id=\"T_d9d4b_row9_col5\" class=\"data row9 col5\" >www.audible.com/pd/B00JS9K7OI</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row10_col0\" class=\"data row10 col0\" >The Farseer: Assassin's Apprentice</td>\n",
       "                        <td id=\"T_d9d4b_row10_col1\" class=\"data row10 col1\" >Robin Hobb</td>\n",
       "                        <td id=\"T_d9d4b_row10_col2\" class=\"data row10 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row10_col3\" class=\"data row10 col3\" >12069</td>\n",
       "                        <td id=\"T_d9d4b_row10_col4\" class=\"data row10 col4\" >B003AO3P5A</td>\n",
       "                        <td id=\"T_d9d4b_row10_col5\" class=\"data row10 col5\" >www.audible.com/pd/B003AO3P5A</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row11_col0\" class=\"data row11 col0\" >Find Her</td>\n",
       "                        <td id=\"T_d9d4b_row11_col1\" class=\"data row11 col1\" >Lisa Gardner</td>\n",
       "                        <td id=\"T_d9d4b_row11_col2\" class=\"data row11 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row11_col3\" class=\"data row11 col3\" >12062</td>\n",
       "                        <td id=\"T_d9d4b_row11_col4\" class=\"data row11 col4\" >B019WKOU18</td>\n",
       "                        <td id=\"T_d9d4b_row11_col5\" class=\"data row11 col5\" >www.audible.com/pd/B019WKOU18</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row12_col0\" class=\"data row12 col0\" >Second Hand Curses</td>\n",
       "                        <td id=\"T_d9d4b_row12_col1\" class=\"data row12 col1\" >Drew Hayes</td>\n",
       "                        <td id=\"T_d9d4b_row12_col2\" class=\"data row12 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row12_col3\" class=\"data row12 col3\" >11560</td>\n",
       "                        <td id=\"T_d9d4b_row12_col4\" class=\"data row12 col4\" >B074XK2R2X</td>\n",
       "                        <td id=\"T_d9d4b_row12_col5\" class=\"data row12 col5\" >www.audible.com/pd/B074XK2R2X</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row13_col0\" class=\"data row13 col0\" >Swan Song</td>\n",
       "                        <td id=\"T_d9d4b_row13_col1\" class=\"data row13 col1\" >Robert R. McCammon</td>\n",
       "                        <td id=\"T_d9d4b_row13_col2\" class=\"data row13 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row13_col3\" class=\"data row13 col3\" >11261</td>\n",
       "                        <td id=\"T_d9d4b_row13_col4\" class=\"data row13 col4\" >B00656GCH8</td>\n",
       "                        <td id=\"T_d9d4b_row13_col5\" class=\"data row13 col5\" >www.audible.com/pd/B00656GCH8</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_d9d4b_row14_col0\" class=\"data row14 col0\" >The Stone Sky</td>\n",
       "                        <td id=\"T_d9d4b_row14_col1\" class=\"data row14 col1\" >N. K. Jemisin</td>\n",
       "                        <td id=\"T_d9d4b_row14_col2\" class=\"data row14 col2\" >4.5</td>\n",
       "                        <td id=\"T_d9d4b_row14_col3\" class=\"data row14 col3\" >11059</td>\n",
       "                        <td id=\"T_d9d4b_row14_col4\" class=\"data row14 col4\" >B07288JYYG</td>\n",
       "                        <td id=\"T_d9d4b_row14_col5\" class=\"data row14 col5\" >www.audible.com/pd/B07288JYYG</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x2cd26631d30>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''' 4.5 star books, top 20'''\n",
    "df_books.sort_values('ratings',ascending=False).loc[df_books['stars'] == 4.5].head(20).style.hide_index().format({'stars': \"{:.2}\"})"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "63fd5069d213b44bf678585dea6b12cceca9941eaf7f819626cde1f2670de90d"
  },
  "kernelspec": {
   "display_name": "Python 3.9.0 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}