Scrapping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import requests\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "home_url = 'https://www.simplytarot.com'\n",
    "base_urls = [\n",
    "    'https://www.simplytarot.com/tarot-card-meanings/major-arcana-tarot-card-meanings/',\n",
    "    'https://www.simplytarot.com/tarot-card-meanings/minor-arcana/swords/',\n",
    "    'https://www.simplytarot.com/tarot-card-meanings/minor-arcana/cups/',\n",
    "    'https://www.simplytarot.com/tarot-card-meanings/minor-arcana/wands/',\n",
    "    'https://www.simplytarot.com/tarot-card-meanings/minor-arcana/pentacles/'\n",
    "]\n",
    "headers={'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "bodies = {}\n",
    "for url in base_urls:\n",
    "    html_body = requests.get(\n",
    "        url,\n",
    "        headers=headers).content\n",
    "    bodies[url] = BeautifulSoup(html_body).find_all('div', class_='cardContainer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "cards = {}\n",
    "errors = []\n",
    "for url, body in bodies.items():\n",
    "    for card in body:\n",
    "        card_name = card.find('img')['alt'].lower().replace(' ', '-').replace('care', 'card').replace('taort', 'tarot')\n",
    "        card_name = card_name.replace('the-', '').replace('.', '-').replace('-tarot-card', '')\n",
    "        response = requests.get(url + card_name, headers=headers)\n",
    "        \n",
    "        if response.status_code > 300:\n",
    "            errors.append(response)\n",
    "        else:\n",
    "            cards[card_name] = response.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "cards_formatted = []\n",
    "for card, body in cards.items():\n",
    "    card_bs = BeautifulSoup(body)\n",
    "    \n",
    "    content = [c for c in card_bs.find_all('div', class_='card') if c.get_text() != ''][0]\n",
    "    \n",
    "    if content != None:\n",
    "        image_url = 'https://www.simplytarot.com' + card_bs.find('', class_='cardImgLink').img['src']\n",
    "        parragraphs = content.find_all('p')\n",
    "    \n",
    "        cards_formatted.append({\n",
    "            'name': card,\n",
    "            'summary': parragraphs[0].get_text(),\n",
    "            'full_meaning': parragraphs[2].get_text(),\n",
    "            'image': image_url\n",
    "        })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_json(obj, file_name):\n",
    "    with open(file_name, 'w') as file:\n",
    "        json.dump(obj, file)\n",
    "save_json(cards_formatted, 'cards.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the cards\n",
    "cards = json.loads(open('./lib/cards.json', 'r').read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "meanings_url = 'https://labyrinthos.co/blogs/tarot-card-meanings-list'\n",
    "\n",
    "html_body = requests.get(\n",
    "        meanings_url,\n",
    "        headers=headers).content\n",
    "meanings_bs = BeautifulSoup(html_body)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "html_cards = meanings_bs.find_all('div', class_='grid__item large--one-quarter medium--one-third text-center card')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "cards_as_map = { card['name'].replace('\"', ''): card for card in cards }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['the-fool', 'the-magician', 'the-empress', 'the-emperor', 'the-high-priestess', 'the-hierophant', 'the-lovers', 'the-chariot', 'justice', 'temperance', 'strength', 'the-hermit', 'the-wheel-of-fortune', 'the-hanged-man', 'death', 'the-devil', 'the-tower', 'the-star', 'the-moon', 'the-sun', 'judgement', 'the-world', 'ace-of-swords', 'two-of-swords', 'three-of-swords', 'four-of-swords', 'five-of-swords', 'six-of-swords', 'seven-of-swords', 'eight-of-swords', 'nine-of-swords', 'ten-of-swords', 'page-of-swords', 'knight-of-swords', 'queen-of-swords', 'king-of-swords', 'ace-of-cups', 'two-of-cups', 'three-of-cups', 'four-of-cups', 'five-of-cups', 'six-of-cups', 'seven-of-cups', 'eight-of-cups', 'nine-of-cups', 'ten-of-cups', 'page-of-cups', 'knight-of-cups', 'queen-of-cups', 'king-of-cups', 'ace-of-wands', 'two-of-wands', 'three-of-wands', 'four-of-wands', 'five-of-wands', 'six-of-wands', 'seven-of-wands', 'eight-of-wands', 'nine-of-wands', 'ten-of-wands', 'page-of-wands', 'knight-of-wands', 'queen-of-wands', 'king-of-wands', 'ace-of-pentacles', 'two-of-pentacles', 'three-of-pentacles', 'four-of-pentacles', 'five-of-pentacles', 'six-of-pentacles', 'seven-of-pentacles', 'eight-of-pentacles', 'nine-of-pentacles', 'ten-of-pentacles', 'page-of-pentacles', 'knight-of-pentacles', 'queen-of-pentacles', 'king-of-pentacles'])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cards_as_map.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_card_name(card):\n",
    "    return card.find('img')['alt'].split(' - ')[0].lower().replace(' ', '-').replace('-meaning', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_upright_reversed(card):\n",
    "    clean_text = card.find('div', class_='rte rte--indented-images').text.strip()\n",
    "    upright, reversed_ = clean_text.replace('Upright: ', '').split(', Reversed: ')\n",
    "    \n",
    "    return upright, reversed_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "for card in html_cards:\n",
    "    name = get_card_name(card)\n",
    "    \n",
    "    upright, reversed_ = get_upright_reversed(card)\n",
    "    \n",
    "    cards_as_map[name]['upright'] = upright\n",
    "    cards_as_map[name]['reversed'] = reversed_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'seven-of-wands\"',\n",
       " 'summary': 'Buy this deck',\n",
       " 'full_meaning': 'The Subject really needs to think this through; a decision has to be taken regarding this issue – should they come away from this relationship/situation? \\xa0Or should they make a real effort to change their attitude and inject some enthusiasm back into it?',\n",
       " 'image': 'https://www.simplytarot.com/wp-content/uploads/2015/04/seven-of-wands-tarot-card.png',\n",
       " 'upright': 'perseverance, defensive, maintaining control',\n",
       " 'reversed': 'give up, destroyed confidence, overwhelmed'}"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cards_as_map['seven-of-wands']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_json(list(cards_as_map.values()), './lib/cards.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "fix card that have errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option(\"display.max_colwidth\", 10000)\n",
    "\n",
    "df = pd.read_json('./lib/cards.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>summary</th>\n",
       "      <th>full_meaning</th>\n",
       "      <th>image</th>\n",
       "      <th>upright</th>\n",
       "      <th>reversed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>the-fool</td>\n",
       "      <td>Short Meaning:  A new phase in life and an urg...</td>\n",
       "      <td>In-depth Meaning:  Something new is starting f...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>innocence, new beginnings, free spirit</td>\n",
       "      <td>recklessness, taken advantage of, inconsideration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>the-magician</td>\n",
       "      <td>Short meaning:  A crossroads. An opportunity f...</td>\n",
       "      <td>In-depth meaning:  When The Magician appears i...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>willpower, desire, creation, manifestation</td>\n",
       "      <td>trickery, illusions, out of touch</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>the-empress</td>\n",
       "      <td>Short meaning:  Can represent pregnancy, the e...</td>\n",
       "      <td>In-depth meaning:  The Empress tends to appear...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>motherhood, fertility, nature</td>\n",
       "      <td>dependence, smothering, emptiness, nosiness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>the-emperor</td>\n",
       "      <td>Short meaning:  Can represent a father figure,...</td>\n",
       "      <td>In-depth meaning:  This card shows the Subject...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>authority, structure, control, fatherhood</td>\n",
       "      <td>tyranny, rigidity, coldness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the-high-priestess</td>\n",
       "      <td>Short meaning:  Growing potential, hidden tale...</td>\n",
       "      <td>In-depth meaning:  The High Priestess represen...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>intuitive, unconscious, inner voice</td>\n",
       "      <td>lack of center, lost inner voice, repressed fe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 name                                            summary  \\\n",
       "0            the-fool  Short Meaning:  A new phase in life and an urg...   \n",
       "1        the-magician  Short meaning:  A crossroads. An opportunity f...   \n",
       "2         the-empress  Short meaning:  Can represent pregnancy, the e...   \n",
       "3         the-emperor  Short meaning:  Can represent a father figure,...   \n",
       "4  the-high-priestess  Short meaning:  Growing potential, hidden tale...   \n",
       "\n",
       "                                        full_meaning  \\\n",
       "0  In-depth Meaning:  Something new is starting f...   \n",
       "1  In-depth meaning:  When The Magician appears i...   \n",
       "2  In-depth meaning:  The Empress tends to appear...   \n",
       "3  In-depth meaning:  This card shows the Subject...   \n",
       "4  In-depth meaning:  The High Priestess represen...   \n",
       "\n",
       "                                               image  \\\n",
       "0  https://www.simplytarot.com/wp-content/uploads...   \n",
       "1  https://www.simplytarot.com/wp-content/uploads...   \n",
       "2  https://www.simplytarot.com/wp-content/uploads...   \n",
       "3  https://www.simplytarot.com/wp-content/uploads...   \n",
       "4  https://www.simplytarot.com/wp-content/uploads...   \n",
       "\n",
       "                                      upright  \\\n",
       "0      innocence, new beginnings, free spirit   \n",
       "1  willpower, desire, creation, manifestation   \n",
       "2               motherhood, fertility, nature   \n",
       "3   authority, structure, control, fatherhood   \n",
       "4         intuitive, unconscious, inner voice   \n",
       "\n",
       "                                            reversed  \n",
       "0  recklessness, taken advantage of, inconsideration  \n",
       "1                  trickery, illusions, out of touch  \n",
       "2        dependence, smothering, emptiness, nosiness  \n",
       "3                        tyranny, rigidity, coldness  \n",
       "4  lack of center, lost inner voice, repressed fe...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>summary</th>\n",
       "      <th>full_meaning</th>\n",
       "      <th>image</th>\n",
       "      <th>upright</th>\n",
       "      <th>reversed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>78</td>\n",
       "      <td>69</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>queen-of-swords</td>\n",
       "      <td>Buy this deck</td>\n",
       "      <td>In-depth meaning:  The Subject may have starte...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>anxiety, hopelessness, trauma</td>\n",
       "      <td>rebellion, subversiveness, new approaches</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   name        summary  \\\n",
       "count                78             78   \n",
       "unique               78             69   \n",
       "top     queen-of-swords  Buy this deck   \n",
       "freq                  1             10   \n",
       "\n",
       "                                             full_meaning  \\\n",
       "count                                                  78   \n",
       "unique                                                 78   \n",
       "top     In-depth meaning:  The Subject may have starte...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                                    image  \\\n",
       "count                                                  78   \n",
       "unique                                                 78   \n",
       "top     https://www.simplytarot.com/wp-content/uploads...   \n",
       "freq                                                    1   \n",
       "\n",
       "                              upright  \\\n",
       "count                              78   \n",
       "unique                             78   \n",
       "top     anxiety, hopelessness, trauma   \n",
       "freq                                1   \n",
       "\n",
       "                                         reversed  \n",
       "count                                          78  \n",
       "unique                                         78  \n",
       "top     rebellion, subversiveness, new approaches  \n",
       "freq                                            1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>summary</th>\n",
       "      <th>full_meaning</th>\n",
       "      <th>image</th>\n",
       "      <th>upright</th>\n",
       "      <th>reversed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>the-devil</td>\n",
       "      <td>Buy this deck</td>\n",
       "      <td>It is time to deal with the situation rather t...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>addiction, materialism, playfulness</td>\n",
       "      <td>freedom, release, restoring control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>two-of-swords</td>\n",
       "      <td>Buy this deck</td>\n",
       "      <td>The Subject is not happy at all and this card ...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>difficult choices, indecision, stalemate</td>\n",
       "      <td>lesser of two evils, no right choice, confusion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>four-of-swords</td>\n",
       "      <td>Buy this deck</td>\n",
       "      <td>Taking time to consider the past and recognisi...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>rest, restoration, contemplation</td>\n",
       "      <td>restlessness, burnout, stress</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>seven-of-swords</td>\n",
       "      <td>Buy this deck</td>\n",
       "      <td>The Subject, or someone around them, may be re...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>deception, trickery, tactics and strategy</td>\n",
       "      <td>coming clean, rethinking approach, deception</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>ten-of-swords</td>\n",
       "      <td>Buy this deck</td>\n",
       "      <td>The message with this card is that, now the tr...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>failure, collapse, defeat</td>\n",
       "      <td>can't get worse, only upwards, inevitable end</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               name        summary  \\\n",
       "15        the-devil  Buy this deck   \n",
       "23    two-of-swords  Buy this deck   \n",
       "25   four-of-swords  Buy this deck   \n",
       "28  seven-of-swords  Buy this deck   \n",
       "31    ten-of-swords  Buy this deck   \n",
       "\n",
       "                                         full_meaning  \\\n",
       "15  It is time to deal with the situation rather t...   \n",
       "23  The Subject is not happy at all and this card ...   \n",
       "25  Taking time to consider the past and recognisi...   \n",
       "28  The Subject, or someone around them, may be re...   \n",
       "31  The message with this card is that, now the tr...   \n",
       "\n",
       "                                                image  \\\n",
       "15  https://www.simplytarot.com/wp-content/uploads...   \n",
       "23  https://www.simplytarot.com/wp-content/uploads...   \n",
       "25  https://www.simplytarot.com/wp-content/uploads...   \n",
       "28  https://www.simplytarot.com/wp-content/uploads...   \n",
       "31  https://www.simplytarot.com/wp-content/uploads...   \n",
       "\n",
       "                                      upright  \\\n",
       "15        addiction, materialism, playfulness   \n",
       "23   difficult choices, indecision, stalemate   \n",
       "25           rest, restoration, contemplation   \n",
       "28  deception, trickery, tactics and strategy   \n",
       "31                  failure, collapse, defeat   \n",
       "\n",
       "                                           reversed  \n",
       "15              freedom, release, restoring control  \n",
       "23  lesser of two evils, no right choice, confusion  \n",
       "25                    restlessness, burnout, stress  \n",
       "28     coming clean, rethinking approach, deception  \n",
       "31    can't get worse, only upwards, inevitable end  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cards_with_wrong_summary = df[df.summary == 'Buy this deck']\n",
    "cards_with_wrong_summary.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cezar/.pyenv/versions/3.7.3/lib/python3.7/site-packages/ipykernel_launcher.py:17: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "urls_with_suits = {\n",
    "    'major': 'https://www.simplytarot.com/tarot-card-meanings/major-arcana-tarot-card-meanings/',\n",
    "    'swords': 'https://www.simplytarot.com/tarot-card-meanings/minor-arcana/swords/',\n",
    "    'cups': 'https://www.simplytarot.com/tarot-card-meanings/minor-arcana/cups/',\n",
    "    'wands': 'https://www.simplytarot.com/tarot-card-meanings/minor-arcana/wands/',\n",
    "    'pentacles': 'https://www.simplytarot.com/tarot-card-meanings/minor-arcana/pentacles/'\n",
    "}\n",
    "\n",
    "def url_for_card_name(card_name):\n",
    "    for suit in urls_with_suits.keys():\n",
    "        if suit in card_name:\n",
    "            return urls_with_suits[suit] + card_name\n",
    "        else:\n",
    "            return urls_with_suits['major'] + card_name\n",
    "\n",
    "suits = ['swords', 'wands', 'cus', 'pentacles', 'major']\n",
    "cards_with_wrong_summary['url'] = cards_with_wrong_summary['name'].apply(lambda x: url_for_card_name(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cards_htmls = {}\n",
    "\n",
    "for i, card in cards_with_wrong_summary.iterrows():\n",
    "    response = requests.get(card['url'], headers=headers)\n",
    "    \n",
    "    cards_htmls[card['name']] = BeautifulSoup(response.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cezar/.pyenv/versions/3.7.3/lib/python3.7/site-packages/ipykernel_launcher.py:1: DeprecationWarning: cgi.escape is deprecated, use html.escape instead\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'&lt;div&gt;lala&lt;/div&gt;'"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cgi.escape(\"<div>lala</div>\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "content = {}\n",
    "for card, body in cards_htmls.items():\n",
    "    card_content = [c for c in body.find_all('div', class_='card') if c.get_text() != ''][0]\n",
    "    \n",
    "    summary = [div for div in card_content.find_all('div') if div.get_text() != ''][0]\n",
    "    full_meaning = card_content.find_all('p')[1]\n",
    "\n",
    "    content[card] = {\n",
    "        'summary': summary,\n",
    "        'full_meaning': full_meaning\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summary</th>\n",
       "      <th>full_meaning</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>the-devil</th>\n",
       "      <td>[[Short meaning:  ], This card points to addic...</td>\n",
       "      <td>[[In-depth meaning:  ], The Devil is an inward...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>two-of-swords</th>\n",
       "      <td>[ , [Short meaning:  ], The Two of Swords show...</td>\n",
       "      <td>[[In-depth meaning:  ], When the Two of Swords...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four-of-swords</th>\n",
       "      <td>[ , [Short meaning:  ], The Four of Swords enc...</td>\n",
       "      <td>[[In-depth meaning:  ], The whole of the Sword...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seven-of-swords</th>\n",
       "      <td>[ , [Short meaning:  ], The Seven of Swords wa...</td>\n",
       "      <td>[[In-depth meaning:  ], The Subject should be ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ten-of-swords</th>\n",
       "      <td>[ , [Short meaning:  ], A difficult situation ...</td>\n",
       "      <td>[[In-depth meaning:  ], The suffering of the S...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                           summary  \\\n",
       "the-devil        [[Short meaning:  ], This card points to addic...   \n",
       "two-of-swords    [ , [Short meaning:  ], The Two of Swords show...   \n",
       "four-of-swords   [ , [Short meaning:  ], The Four of Swords enc...   \n",
       "seven-of-swords  [ , [Short meaning:  ], The Seven of Swords wa...   \n",
       "ten-of-swords    [ , [Short meaning:  ], A difficult situation ...   \n",
       "\n",
       "                                                      full_meaning  \n",
       "the-devil        [[In-depth meaning:  ], The Devil is an inward...  \n",
       "two-of-swords    [[In-depth meaning:  ], When the Two of Swords...  \n",
       "four-of-swords   [[In-depth meaning:  ], The whole of the Sword...  \n",
       "seven-of-swords  [[In-depth meaning:  ], The Subject should be ...  \n",
       "ten-of-swords    [[In-depth meaning:  ], The suffering of the S...  "
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content_df = pd.DataFrame(content.values(),index=content.keys())\n",
    "content_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cezar/.pyenv/versions/3.7.3/lib/python3.7/site-packages/pandas/core/indexing.py:671: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n",
      "/home/cezar/.pyenv/versions/3.7.3/lib/python3.7/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "cards_with_wrong_summary.loc[content.keys(), 'summary'] = content_df['summary']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>summary</th>\n",
       "      <th>full_meaning</th>\n",
       "      <th>image</th>\n",
       "      <th>upright</th>\n",
       "      <th>reversed</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>the-devil</th>\n",
       "      <td>the-devil</td>\n",
       "      <td>[[Short meaning:  ], This card points to addic...</td>\n",
       "      <td>It is time to deal with the situation rather t...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>addiction, materialism, playfulness</td>\n",
       "      <td>freedom, release, restoring control</td>\n",
       "      <td>https://www.simplytarot.com/tarot-card-meaning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>two-of-swords</th>\n",
       "      <td>two-of-swords</td>\n",
       "      <td>[ , [Short meaning:  ], The Two of Swords show...</td>\n",
       "      <td>The Subject is not happy at all and this card ...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>difficult choices, indecision, stalemate</td>\n",
       "      <td>lesser of two evils, no right choice, confusion</td>\n",
       "      <td>https://www.simplytarot.com/tarot-card-meaning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four-of-swords</th>\n",
       "      <td>four-of-swords</td>\n",
       "      <td>[ , [Short meaning:  ], The Four of Swords enc...</td>\n",
       "      <td>Taking time to consider the past and recognisi...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>rest, restoration, contemplation</td>\n",
       "      <td>restlessness, burnout, stress</td>\n",
       "      <td>https://www.simplytarot.com/tarot-card-meaning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seven-of-swords</th>\n",
       "      <td>seven-of-swords</td>\n",
       "      <td>[ , [Short meaning:  ], The Seven of Swords wa...</td>\n",
       "      <td>The Subject, or someone around them, may be re...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>deception, trickery, tactics and strategy</td>\n",
       "      <td>coming clean, rethinking approach, deception</td>\n",
       "      <td>https://www.simplytarot.com/tarot-card-meaning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ten-of-swords</th>\n",
       "      <td>ten-of-swords</td>\n",
       "      <td>[ , [Short meaning:  ], A difficult situation ...</td>\n",
       "      <td>The message with this card is that, now the tr...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>failure, collapse, defeat</td>\n",
       "      <td>can't get worse, only upwards, inevitable end</td>\n",
       "      <td>https://www.simplytarot.com/tarot-card-meaning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ace-of-wands</th>\n",
       "      <td>ace-of-wands</td>\n",
       "      <td>[ , [Short meaning:  ], The Ace of Wands shows...</td>\n",
       "      <td>The Ace of Wands is the birth of a vision, a n...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>creation, willpower, inspiration, desire</td>\n",
       "      <td>lack of energy, lack of passion, boredom</td>\n",
       "      <td>https://www.simplytarot.com/tarot-card-meaning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seven-of-wands\"</th>\n",
       "      <td>seven-of-wands\"</td>\n",
       "      <td>[ , [Short meaning:  ], The Seven of Wands sho...</td>\n",
       "      <td>The Subject really needs to think this through...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>perseverance, defensive, maintaining control</td>\n",
       "      <td>give up, destroyed confidence, overwhelmed</td>\n",
       "      <td>https://www.simplytarot.com/tarot-card-meaning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eight-of-wands</th>\n",
       "      <td>eight-of-wands</td>\n",
       "      <td>[ , [Short meaning:  ], The Eight of Wands sho...</td>\n",
       "      <td>This situation hasn’t always been so positive,...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>rapid action, movement, quick decisions</td>\n",
       "      <td>panic, waiting, slowdown</td>\n",
       "      <td>https://www.simplytarot.com/tarot-card-meaning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nine-of-wands</th>\n",
       "      <td>nine-of-wands</td>\n",
       "      <td>[ , [Short meaning: ], The Nine of Wands is th...</td>\n",
       "      <td>The Subject would benefit from looking at this...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>resilience, grit, last stand</td>\n",
       "      <td>exhaustion, fatigue, questioning motivations</td>\n",
       "      <td>https://www.simplytarot.com/tarot-card-meaning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ten-of-wands</th>\n",
       "      <td>ten-of-wands</td>\n",
       "      <td>[ , [Short meaning:  ], The Ten of Wands shows...</td>\n",
       "      <td>The Subject may also feel overwhelmed by someo...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>accomplishment, responsibility, burden</td>\n",
       "      <td>inability to delegate, overstressed, burnt out</td>\n",
       "      <td>https://www.simplytarot.com/tarot-card-meaning...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            name  \\\n",
       "name                               \n",
       "the-devil              the-devil   \n",
       "two-of-swords      two-of-swords   \n",
       "four-of-swords    four-of-swords   \n",
       "seven-of-swords  seven-of-swords   \n",
       "ten-of-swords      ten-of-swords   \n",
       "ace-of-wands        ace-of-wands   \n",
       "seven-of-wands\"  seven-of-wands\"   \n",
       "eight-of-wands    eight-of-wands   \n",
       "nine-of-wands      nine-of-wands   \n",
       "ten-of-wands        ten-of-wands   \n",
       "\n",
       "                                                           summary  \\\n",
       "name                                                                 \n",
       "the-devil        [[Short meaning:  ], This card points to addic...   \n",
       "two-of-swords    [ , [Short meaning:  ], The Two of Swords show...   \n",
       "four-of-swords   [ , [Short meaning:  ], The Four of Swords enc...   \n",
       "seven-of-swords  [ , [Short meaning:  ], The Seven of Swords wa...   \n",
       "ten-of-swords    [ , [Short meaning:  ], A difficult situation ...   \n",
       "ace-of-wands     [ , [Short meaning:  ], The Ace of Wands shows...   \n",
       "seven-of-wands\"  [ , [Short meaning:  ], The Seven of Wands sho...   \n",
       "eight-of-wands   [ , [Short meaning:  ], The Eight of Wands sho...   \n",
       "nine-of-wands    [ , [Short meaning: ], The Nine of Wands is th...   \n",
       "ten-of-wands     [ , [Short meaning:  ], The Ten of Wands shows...   \n",
       "\n",
       "                                                      full_meaning  \\\n",
       "name                                                                 \n",
       "the-devil        It is time to deal with the situation rather t...   \n",
       "two-of-swords    The Subject is not happy at all and this card ...   \n",
       "four-of-swords   Taking time to consider the past and recognisi...   \n",
       "seven-of-swords  The Subject, or someone around them, may be re...   \n",
       "ten-of-swords    The message with this card is that, now the tr...   \n",
       "ace-of-wands     The Ace of Wands is the birth of a vision, a n...   \n",
       "seven-of-wands\"  The Subject really needs to think this through...   \n",
       "eight-of-wands   This situation hasn’t always been so positive,...   \n",
       "nine-of-wands    The Subject would benefit from looking at this...   \n",
       "ten-of-wands     The Subject may also feel overwhelmed by someo...   \n",
       "\n",
       "                                                             image  \\\n",
       "name                                                                 \n",
       "the-devil        https://www.simplytarot.com/wp-content/uploads...   \n",
       "two-of-swords    https://www.simplytarot.com/wp-content/uploads...   \n",
       "four-of-swords   https://www.simplytarot.com/wp-content/uploads...   \n",
       "seven-of-swords  https://www.simplytarot.com/wp-content/uploads...   \n",
       "ten-of-swords    https://www.simplytarot.com/wp-content/uploads...   \n",
       "ace-of-wands     https://www.simplytarot.com/wp-content/uploads...   \n",
       "seven-of-wands\"  https://www.simplytarot.com/wp-content/uploads...   \n",
       "eight-of-wands   https://www.simplytarot.com/wp-content/uploads...   \n",
       "nine-of-wands    https://www.simplytarot.com/wp-content/uploads...   \n",
       "ten-of-wands     https://www.simplytarot.com/wp-content/uploads...   \n",
       "\n",
       "                                                      upright  \\\n",
       "name                                                            \n",
       "the-devil                 addiction, materialism, playfulness   \n",
       "two-of-swords        difficult choices, indecision, stalemate   \n",
       "four-of-swords               rest, restoration, contemplation   \n",
       "seven-of-swords     deception, trickery, tactics and strategy   \n",
       "ten-of-swords                       failure, collapse, defeat   \n",
       "ace-of-wands         creation, willpower, inspiration, desire   \n",
       "seven-of-wands\"  perseverance, defensive, maintaining control   \n",
       "eight-of-wands        rapid action, movement, quick decisions   \n",
       "nine-of-wands                    resilience, grit, last stand   \n",
       "ten-of-wands           accomplishment, responsibility, burden   \n",
       "\n",
       "                                                        reversed  \\\n",
       "name                                                               \n",
       "the-devil                    freedom, release, restoring control   \n",
       "two-of-swords    lesser of two evils, no right choice, confusion   \n",
       "four-of-swords                     restlessness, burnout, stress   \n",
       "seven-of-swords     coming clean, rethinking approach, deception   \n",
       "ten-of-swords      can't get worse, only upwards, inevitable end   \n",
       "ace-of-wands            lack of energy, lack of passion, boredom   \n",
       "seven-of-wands\"       give up, destroyed confidence, overwhelmed   \n",
       "eight-of-wands                          panic, waiting, slowdown   \n",
       "nine-of-wands       exhaustion, fatigue, questioning motivations   \n",
       "ten-of-wands      inability to delegate, overstressed, burnt out   \n",
       "\n",
       "                                                               url  \n",
       "name                                                                \n",
       "the-devil        https://www.simplytarot.com/tarot-card-meaning...  \n",
       "two-of-swords    https://www.simplytarot.com/tarot-card-meaning...  \n",
       "four-of-swords   https://www.simplytarot.com/tarot-card-meaning...  \n",
       "seven-of-swords  https://www.simplytarot.com/tarot-card-meaning...  \n",
       "ten-of-swords    https://www.simplytarot.com/tarot-card-meaning...  \n",
       "ace-of-wands     https://www.simplytarot.com/tarot-card-meaning...  \n",
       "seven-of-wands\"  https://www.simplytarot.com/tarot-card-meaning...  \n",
       "eight-of-wands   https://www.simplytarot.com/tarot-card-meaning...  \n",
       "nine-of-wands    https://www.simplytarot.com/tarot-card-meaning...  \n",
       "ten-of-wands     https://www.simplytarot.com/tarot-card-meaning...  "
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cards_with_wrong_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>summary</th>\n",
       "      <th>full_meaning</th>\n",
       "      <th>image</th>\n",
       "      <th>upright</th>\n",
       "      <th>reversed</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>the-fool</th>\n",
       "      <td>the-fool</td>\n",
       "      <td>Short Meaning:  A new phase in life and an urg...</td>\n",
       "      <td>In-depth Meaning:  Something new is starting f...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>innocence, new beginnings, free spirit</td>\n",
       "      <td>recklessness, taken advantage of, inconsideration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the-magician</th>\n",
       "      <td>the-magician</td>\n",
       "      <td>Short meaning:  A crossroads. An opportunity f...</td>\n",
       "      <td>In-depth meaning:  When The Magician appears i...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>willpower, desire, creation, manifestation</td>\n",
       "      <td>trickery, illusions, out of touch</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the-empress</th>\n",
       "      <td>the-empress</td>\n",
       "      <td>Short meaning:  Can represent pregnancy, the e...</td>\n",
       "      <td>In-depth meaning:  The Empress tends to appear...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>motherhood, fertility, nature</td>\n",
       "      <td>dependence, smothering, emptiness, nosiness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the-emperor</th>\n",
       "      <td>the-emperor</td>\n",
       "      <td>Short meaning:  Can represent a father figure,...</td>\n",
       "      <td>In-depth meaning:  This card shows the Subject...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>authority, structure, control, fatherhood</td>\n",
       "      <td>tyranny, rigidity, coldness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the-high-priestess</th>\n",
       "      <td>the-high-priestess</td>\n",
       "      <td>Short meaning:  Growing potential, hidden tale...</td>\n",
       "      <td>In-depth meaning:  The High Priestess represen...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>intuitive, unconscious, inner voice</td>\n",
       "      <td>lack of center, lost inner voice, repressed fe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  name  \\\n",
       "name                                     \n",
       "the-fool                      the-fool   \n",
       "the-magician              the-magician   \n",
       "the-empress                the-empress   \n",
       "the-emperor                the-emperor   \n",
       "the-high-priestess  the-high-priestess   \n",
       "\n",
       "                                                              summary  \\\n",
       "name                                                                    \n",
       "the-fool            Short Meaning:  A new phase in life and an urg...   \n",
       "the-magician        Short meaning:  A crossroads. An opportunity f...   \n",
       "the-empress         Short meaning:  Can represent pregnancy, the e...   \n",
       "the-emperor         Short meaning:  Can represent a father figure,...   \n",
       "the-high-priestess  Short meaning:  Growing potential, hidden tale...   \n",
       "\n",
       "                                                         full_meaning  \\\n",
       "name                                                                    \n",
       "the-fool            In-depth Meaning:  Something new is starting f...   \n",
       "the-magician        In-depth meaning:  When The Magician appears i...   \n",
       "the-empress         In-depth meaning:  The Empress tends to appear...   \n",
       "the-emperor         In-depth meaning:  This card shows the Subject...   \n",
       "the-high-priestess  In-depth meaning:  The High Priestess represen...   \n",
       "\n",
       "                                                                image  \\\n",
       "name                                                                    \n",
       "the-fool            https://www.simplytarot.com/wp-content/uploads...   \n",
       "the-magician        https://www.simplytarot.com/wp-content/uploads...   \n",
       "the-empress         https://www.simplytarot.com/wp-content/uploads...   \n",
       "the-emperor         https://www.simplytarot.com/wp-content/uploads...   \n",
       "the-high-priestess  https://www.simplytarot.com/wp-content/uploads...   \n",
       "\n",
       "                                                       upright  \\\n",
       "name                                                             \n",
       "the-fool                innocence, new beginnings, free spirit   \n",
       "the-magician        willpower, desire, creation, manifestation   \n",
       "the-empress                      motherhood, fertility, nature   \n",
       "the-emperor          authority, structure, control, fatherhood   \n",
       "the-high-priestess         intuitive, unconscious, inner voice   \n",
       "\n",
       "                                                             reversed  \n",
       "name                                                                   \n",
       "the-fool            recklessness, taken advantage of, inconsideration  \n",
       "the-magician                        trickery, illusions, out of touch  \n",
       "the-empress               dependence, smothering, emptiness, nosiness  \n",
       "the-emperor                               tyranny, rigidity, coldness  \n",
       "the-high-priestess  lack of center, lost inner voice, repressed fe...  "
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.index = df['name']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[cards_with_wrong_summary.index, 'summary'] = cards_with_wrong_summary['summary']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>summary</th>\n",
       "      <th>full_meaning</th>\n",
       "      <th>image</th>\n",
       "      <th>upright</th>\n",
       "      <th>reversed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>queen-of-swords</td>\n",
       "      <td>Short meaning:  The Ace of Swords is a sign of...</td>\n",
       "      <td>In-depth meaning:  The Subject may have starte...</td>\n",
       "      <td>https://www.simplytarot.com/wp-content/uploads...</td>\n",
       "      <td>anxiety, hopelessness, trauma</td>\n",
       "      <td>rebellion, subversiveness, new approaches</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   name                                            summary  \\\n",
       "count                78                                                 78   \n",
       "unique               78                                                 78   \n",
       "top     queen-of-swords  Short meaning:  The Ace of Swords is a sign of...   \n",
       "freq                  1                                                  1   \n",
       "\n",
       "                                             full_meaning  \\\n",
       "count                                                  78   \n",
       "unique                                                 78   \n",
       "top     In-depth meaning:  The Subject may have starte...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                                    image  \\\n",
       "count                                                  78   \n",
       "unique                                                 78   \n",
       "top     https://www.simplytarot.com/wp-content/uploads...   \n",
       "freq                                                    1   \n",
       "\n",
       "                              upright  \\\n",
       "count                              78   \n",
       "unique                             78   \n",
       "top     anxiety, hopelessness, trauma   \n",
       "freq                                1   \n",
       "\n",
       "                                         reversed  \n",
       "count                                          78  \n",
       "unique                                         78  \n",
       "top     rebellion, subversiveness, new approaches  \n",
       "freq                                            1  "
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fix the image link of the ten of swords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df['name'] == 'ten-of-swords', 'image'] = 'https://www.simplytarot.com/wp-content/uploads/2015/04/ten-of-swords-tarot-card.png'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name\n",
       "ten-of-swords    https://www.simplytarot.com/wp-content/uploads/2015/04/ten-of-swords-tarot-card.png\n",
       "Name: image, dtype: object"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['name'] == 'ten-of-swords']['image']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_json('./lib/cards.json', orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name\n",
       "the-fool                          https://www.simplytarot.com/wp-content/uploads/2015/04/the-fool-tarot-card.png\n",
       "the-magician                  https://www.simplytarot.com/wp-content/uploads/2015/04/the-magician-tarot-card.png\n",
       "the-empress                    https://www.simplytarot.com/wp-content/uploads/2015/04/the-empress-tarot-card.png\n",
       "the-emperor                    https://www.simplytarot.com/wp-content/uploads/2015/04/the-emperor-tarot-card.png\n",
       "the-high-priestess      https://www.simplytarot.com/wp-content/uploads/2015/04/the-high-priestess-tarot-card.png\n",
       "                                                                 ...                                            \n",
       "ten-of-pentacles          https://www.simplytarot.com/wp-content/uploads/2015/04/ten-of-pentacles-tarot-card.png\n",
       "page-of-pentacles        https://www.simplytarot.com/wp-content/uploads/2015/04/page-of-pentacles-tarot-card.png\n",
       "knight-of-pentacles    https://www.simplytarot.com/wp-content/uploads/2015/04/knight-of-pentacles-tarot-card.png\n",
       "queen-of-pentacles      https://www.simplytarot.com/wp-content/uploads/2015/04/queen-of-pentacles-tarot-card.png\n",
       "king-of-pentacles                   https://www.simplytarot.com/wp-content/uploads/2015/04/King-of-Pentacles.png\n",
       "Name: image, Length: 78, dtype: object"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clean strings\n",
    "df['image'] = df['image'].apply(lambda x: x.replace(\"'\", '').strip('b'))\n",
    "df['image']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_json('./lib/cards.json', orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>summary</th>\n",
       "      <th>full_meaning</th>\n",
       "      <th>image</th>\n",
       "      <th>upright</th>\n",
       "      <th>reversed</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>the-fool</th>\n",
       "      <td>the-fool</td>\n",
       "      <td>A new phase in life and an urge to explore new...</td>\n",
       "      <td>Something new is starting for the Subject whic...</td>\n",
       "      <td>https://tarot-api.s3.amazonaws.com/images/majo...</td>\n",
       "      <td>innocence, new beginnings, free spirit</td>\n",
       "      <td>recklessness, taken advantage of, inconsideration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the-magician</th>\n",
       "      <td>the-magician</td>\n",
       "      <td>A crossroads. An opportunity for constructive ...</td>\n",
       "      <td>When The Magician appears it is time for the S...</td>\n",
       "      <td>https://tarot-api.s3.amazonaws.com/images/majo...</td>\n",
       "      <td>willpower, desire, creation, manifestation</td>\n",
       "      <td>trickery, illusions, out of touch</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the-empress</th>\n",
       "      <td>the-empress</td>\n",
       "      <td>Can represent pregnancy, the experience of mot...</td>\n",
       "      <td>The Empress tends to appear where relationship...</td>\n",
       "      <td>https://tarot-api.s3.amazonaws.com/images/majo...</td>\n",
       "      <td>motherhood, fertility, nature</td>\n",
       "      <td>dependence, smothering, emptiness, nosiness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the-emperor</th>\n",
       "      <td>the-emperor</td>\n",
       "      <td>Can represent a father figure, an older man or...</td>\n",
       "      <td>This card shows the Subject using their skills...</td>\n",
       "      <td>https://tarot-api.s3.amazonaws.com/images/majo...</td>\n",
       "      <td>authority, structure, control, fatherhood</td>\n",
       "      <td>tyranny, rigidity, coldness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the-high-priestess</th>\n",
       "      <td>the-high-priestess</td>\n",
       "      <td>Growing potential, hidden talents or an unknow...</td>\n",
       "      <td>The High Priestess represents secrets and thin...</td>\n",
       "      <td>https://tarot-api.s3.amazonaws.com/images/majo...</td>\n",
       "      <td>intuitive, unconscious, inner voice</td>\n",
       "      <td>lack of center, lost inner voice, repressed fe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  name  \\\n",
       "name                                     \n",
       "the-fool                      the-fool   \n",
       "the-magician              the-magician   \n",
       "the-empress                the-empress   \n",
       "the-emperor                the-emperor   \n",
       "the-high-priestess  the-high-priestess   \n",
       "\n",
       "                                                              summary  \\\n",
       "name                                                                    \n",
       "the-fool            A new phase in life and an urge to explore new...   \n",
       "the-magician        A crossroads. An opportunity for constructive ...   \n",
       "the-empress         Can represent pregnancy, the experience of mot...   \n",
       "the-emperor         Can represent a father figure, an older man or...   \n",
       "the-high-priestess  Growing potential, hidden talents or an unknow...   \n",
       "\n",
       "                                                         full_meaning  \\\n",
       "name                                                                    \n",
       "the-fool            Something new is starting for the Subject whic...   \n",
       "the-magician        When The Magician appears it is time for the S...   \n",
       "the-empress         The Empress tends to appear where relationship...   \n",
       "the-emperor         This card shows the Subject using their skills...   \n",
       "the-high-priestess  The High Priestess represents secrets and thin...   \n",
       "\n",
       "                                                                image  \\\n",
       "name                                                                    \n",
       "the-fool            https://tarot-api.s3.amazonaws.com/images/majo...   \n",
       "the-magician        https://tarot-api.s3.amazonaws.com/images/majo...   \n",
       "the-empress         https://tarot-api.s3.amazonaws.com/images/majo...   \n",
       "the-emperor         https://tarot-api.s3.amazonaws.com/images/majo...   \n",
       "the-high-priestess  https://tarot-api.s3.amazonaws.com/images/majo...   \n",
       "\n",
       "                                                       upright  \\\n",
       "name                                                             \n",
       "the-fool                innocence, new beginnings, free spirit   \n",
       "the-magician        willpower, desire, creation, manifestation   \n",
       "the-empress                      motherhood, fertility, nature   \n",
       "the-emperor          authority, structure, control, fatherhood   \n",
       "the-high-priestess         intuitive, unconscious, inner voice   \n",
       "\n",
       "                                                             reversed  \n",
       "name                                                                   \n",
       "the-fool            recklessness, taken advantage of, inconsideration  \n",
       "the-magician                        trickery, illusions, out of touch  \n",
       "the-empress               dependence, smothering, emptiness, nosiness  \n",
       "the-emperor                               tyranny, rigidity, coldness  \n",
       "the-high-priestess  lack of center, lost inner voice, repressed fe...  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_json('./lib/cards.json')\n",
    "df.index = df['name']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "cards_names = open('./lib/cards.txt', 'r').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cards_with_numbers = { name.lower().replace(' ', '-'): i for i, name in enumerate(cards_names.split(\"\\n\")) if name != ''}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['card_index'] = df.apply(lambda x: cards_with_numbers[x['name']], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.to_json('./lib/cards.json', orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}