express_feed.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib\n",
    "from urllib.error import HTTPError\n",
    "import requests\n",
    "import ssl\n",
    "ssl._create_default_https_context = ssl._create_unverified_context\n",
    "import csv\n",
    "from xml import etree\n",
    "from xml.sax.saxutils import unescape\n",
    "import xml.etree.ElementTree as ET\n",
    "import json\n",
    "import html\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import unicodedata\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.mode.chained_assignment = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_root(url):\n",
    "    req = urllib.request.Request(url=url)\n",
    "    try:\n",
    "        handler = urllib.request.urlopen(req)\n",
    "        tree = ET.ElementTree(file = handler)\n",
    "        root = tree.getroot()\n",
    "        return root\n",
    "    except HTTPError as e:\n",
    "        content = e.read()\n",
    "        return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = \"https://www.costa-coffee.cz/express.php\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "root = get_root(base_url)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_column_names(root):\n",
    "    tags = []\n",
    "    for i, child in enumerate(root):  \n",
    "        for subchild in child[-1:]: \n",
    "            for k in subchild:\n",
    "                tags.append(k.tag)\n",
    "    return tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = get_column_names(root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(root):\n",
    "    records = []\n",
    "    data = []\n",
    "    for i, child in enumerate(root):\n",
    "        for subchild in child:\n",
    "            for j in subchild:\n",
    "                records.append(j.text)\n",
    "            try:\n",
    "                if len(records) != 0:\n",
    "                    data.append(records)\n",
    "                    records = []\n",
    "            except AttributeError as error:\n",
    "                return error\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = get_data(root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "rawDf = pd.DataFrame(records, columns=tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>category</th>\n",
       "      <th>lat</th>\n",
       "      <th>lng</th>\n",
       "      <th>adresa</th>\n",
       "      <th>guid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Costa Express - Praha 4</td>\n",
       "      <td>Costa Express</td>\n",
       "      <td>50.034589163874585</td>\n",
       "      <td>14.455249451931763</td>\n",
       "      <td>ČS Shell - Praha 4, ulice Vídeňská</td>\n",
       "      <td>https://www.costa-coffee.cz/costa/costa-express</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Costa Express - Mělník</td>\n",
       "      <td>Costa Express</td>\n",
       "      <td>50.34802581272294</td>\n",
       "      <td>14.492174487249713</td>\n",
       "      <td>ČS Shell - Mělník, ulice Mladoboleslavská</td>\n",
       "      <td>https://www.costa-coffee.cz/costa/costa-express</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Costa Express - Praha 6</td>\n",
       "      <td>Costa Express</td>\n",
       "      <td>50.093505335502975</td>\n",
       "      <td>14.331827076074205</td>\n",
       "      <td>ČS Shell - Praha 6, ulice Evropská</td>\n",
       "      <td>https://www.costa-coffee.cz/costa/costa-express</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Costa Express - Rokycany</td>\n",
       "      <td>Costa Express</td>\n",
       "      <td>49.7538998</td>\n",
       "      <td>13.5894548</td>\n",
       "      <td>ČS Shell - Rokycany, dálnice D5 (směr Praha)</td>\n",
       "      <td>https://www.costa-coffee.cz/costa/costa-express</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Costa Express - Tábor</td>\n",
       "      <td>Costa Express</td>\n",
       "      <td>49.41056965127617</td>\n",
       "      <td>14.683676139550812</td>\n",
       "      <td>ČS Shell - Tábor, ulice Chýnovská</td>\n",
       "      <td>https://www.costa-coffee.cz/costa/costa-express</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      title       category                 lat  \\\n",
       "0   Costa Express - Praha 4  Costa Express  50.034589163874585   \n",
       "1    Costa Express - Mělník  Costa Express   50.34802581272294   \n",
       "2   Costa Express - Praha 6  Costa Express  50.093505335502975   \n",
       "3  Costa Express - Rokycany  Costa Express          49.7538998   \n",
       "4     Costa Express - Tábor  Costa Express   49.41056965127617   \n",
       "\n",
       "                  lng                                        adresa  \\\n",
       "0  14.455249451931763            ČS Shell - Praha 4, ulice Vídeňská   \n",
       "1  14.492174487249713     ČS Shell - Mělník, ulice Mladoboleslavská   \n",
       "2  14.331827076074205            ČS Shell - Praha 6, ulice Evropská   \n",
       "3          13.5894548  ČS Shell - Rokycany, dálnice D5 (směr Praha)   \n",
       "4  14.683676139550812             ČS Shell - Tábor, ulice Chýnovská   \n",
       "\n",
       "                                              guid  \n",
       "0  https://www.costa-coffee.cz/costa/costa-express  \n",
       "1  https://www.costa-coffee.cz/costa/costa-express  \n",
       "2  https://www.costa-coffee.cz/costa/costa-express  \n",
       "3  https://www.costa-coffee.cz/costa/costa-express  \n",
       "4  https://www.costa-coffee.cz/costa/costa-express  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rawDf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "expressDf = rawDf[[\"title\", \"lat\", \"lng\", \"adresa\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "expressDf = expressDf.rename(columns={'adresa': 'address'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>lat</th>\n",
       "      <th>lng</th>\n",
       "      <th>address</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Costa Express - Praha 4</td>\n",
       "      <td>50.034589163874585</td>\n",
       "      <td>14.455249451931763</td>\n",
       "      <td>ČS Shell - Praha 4, ulice Vídeňská</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Costa Express - Mělník</td>\n",
       "      <td>50.34802581272294</td>\n",
       "      <td>14.492174487249713</td>\n",
       "      <td>ČS Shell - Mělník, ulice Mladoboleslavská</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Costa Express - Praha 6</td>\n",
       "      <td>50.093505335502975</td>\n",
       "      <td>14.331827076074205</td>\n",
       "      <td>ČS Shell - Praha 6, ulice Evropská</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Costa Express - Rokycany</td>\n",
       "      <td>49.7538998</td>\n",
       "      <td>13.5894548</td>\n",
       "      <td>ČS Shell - Rokycany, dálnice D5 (směr Praha)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Costa Express - Tábor</td>\n",
       "      <td>49.41056965127617</td>\n",
       "      <td>14.683676139550812</td>\n",
       "      <td>ČS Shell - Tábor, ulice Chýnovská</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      title                 lat                 lng  \\\n",
       "0   Costa Express - Praha 4  50.034589163874585  14.455249451931763   \n",
       "1    Costa Express - Mělník   50.34802581272294  14.492174487249713   \n",
       "2   Costa Express - Praha 6  50.093505335502975  14.331827076074205   \n",
       "3  Costa Express - Rokycany          49.7538998          13.5894548   \n",
       "4     Costa Express - Tábor   49.41056965127617  14.683676139550812   \n",
       "\n",
       "                                        address  \n",
       "0            ČS Shell - Praha 4, ulice Vídeňská  \n",
       "1     ČS Shell - Mělník, ulice Mladoboleslavská  \n",
       "2            ČS Shell - Praha 6, ulice Evropská  \n",
       "3  ČS Shell - Rokycany, dálnice D5 (směr Praha)  \n",
       "4             ČS Shell - Tábor, ulice Chýnovská  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expressDf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_to_json(df):\n",
    "    express_json = df.to_json(orient='records')\n",
    "    temp_po = json.loads(express_json, encoding='utf-8')\n",
    "    temp_json = json.dumps(temp_po, ensure_ascii = False)\n",
    "    return temp_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = df_to_json(expressDf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_json(json):\n",
    "    return print(json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{\"title\": \"Costa Express - Praha 4\", \"lat\": \"50.034589163874585\", \"lng\": \"14.455249451931763\", \"address\": \"ČS Shell - Praha 4, ulice Vídeňská\"}, {\"title\": \"Costa Express - Mělník\", \"lat\": \"50.34802581272294\", \"lng\": \"14.492174487249713\", \"address\": \"ČS Shell - Mělník, ulice Mladoboleslavská\"}, {\"title\": \"Costa Express - Praha 6\", \"lat\": \"50.093505335502975\", \"lng\": \"14.331827076074205\", \"address\": \"ČS Shell - Praha 6, ulice Evropská\"}, {\"title\": \"Costa Express - Rokycany\", \"lat\": \"49.7538998\", \"lng\": \"13.5894548\", \"address\": \"ČS Shell - Rokycany, dálnice D5 (směr Praha)\"}, {\"title\": \"Costa Express - Tábor\", \"lat\": \"49.41056965127617\", \"lng\": \"14.683676139550812\", \"address\": \"ČS Shell - Tábor, ulice Chýnovská\"}, {\"title\": \"Costa Express - Velké Meziříčí\", \"lat\": \"49.35369844640703\", \"lng\": \"16.033142081103506\", \"address\": \"ČS Shell - Velké Meziříčí, dálnice D1 (směr Brno – Praha)\"}, {\"title\": \"Costa Express - Praha\", \"lat\": \"50.1089157\", \"lng\": \"14.6410117\", \"address\": \"ČS Shell - Dálnice D11, Horní Počernice (směr Praha)\"}, {\"title\": \"Costa Express - Praha\", \"lat\": \"50.1123864\", \"lng\": \"14.2822112\", \"address\": \"ČS Shell - Letiště Václava Havla, Praha (u výjezdu směr Praha)\"}, {\"title\": \"Costa Express - Dobříš\", \"lat\": \"49.79767630000001\", \"lng\": \"14.1955765\", \"address\": \"ČS Shell - Rychlostní silnice R4, Dobříš (směr Dobříš)\"}, {\"title\": \"Costa Express - Kalná\", \"lat\": \"49.7136513\", \"lng\": \"15.0715787\", \"address\": \"ČS Shell - Dálnice D1, Kalná (směr Praha)\"}, {\"title\": \"Costa Express - Humpolec\", \"lat\": \"49.5415238\", \"lng\": \"15.3593232\", \"address\": \"ČS Shell - Humpolec (směr Praha) \"}, {\"title\": \"Costa Express - Popůvky\", \"lat\": \"49.1953012\", \"lng\": \"16.49182\", \"address\": \"ČS Shell - Popůvky (směr Praha)\"}, {\"title\": \"Costa Express - Měřín\", \"lat\": \"49.3777304\", \"lng\": \"15.9461842\", \"address\": \"ČS Shell - Měřín (směr Brno)\"}, {\"title\": \"Costa Express - Příšovice\", \"lat\": \"50.579117\", \"lng\": \"15.0704825\", \"address\": \"ČS Shell - Rychlostní silnice D10, Příšovice (v obou směrech)\"}, {\"title\": \"Costa Express - Ladná\", \"lat\": \"48.8094618\", \"lng\": \"16.8938396\", \"address\": \"ČS Shell - Dálnice D2, Ladná (v obou směrech)\"}, {\"title\": \"Costa Express - Velký Újezd\", \"lat\": \"49.5803313\", \"lng\": \"17.4611068\", \"address\": \"ČS Shell - Rychlostní silnice D35, Velký Újezd (směr Ostrava)\"}, {\"title\": \"Costa Express - Pustiměř\", \"lat\": \"49.31802889999999\", \"lng\": \"17.0384517\", \"address\": \"ČS Shell - Rychlostní silnice D46, Pustiměř (směr Brno)\"}, {\"title\": \"Costa Express - Svojkovice\", \"lat\": \"49.7572322\", \"lng\": \"13.6169867\", \"address\": \"ČS Shell - Svojkovice, dálnice D5 (směr Plzeň)\"}, {\"title\": \"Costa Express - Stochov\", \"lat\": \"50.1571417\", \"lng\": \"13.9694495\", \"address\": \"ČS Shell - Rychlostní silnice D6, Stochov (směr Praha)\"}, {\"title\": \"Costa Express - Vražné\", \"lat\": \"49.6314931\", \"lng\": \"17.8432091\", \"address\": \"ČS Shell - Dálnice D47, Vražné (v obou směrech)\"}, {\"title\": \"Costa Express - Málkovice\", \"lat\": \"49.74620179999999\", \"lng\": \"12.8416096\", \"address\": \"ČS Shell - Dálnice D5, Málkovice (v obou směrech)\\n\"}, {\"title\": \"Costa Express - Všechlapy\", \"lat\": \"49.7802058\", \"lng\": \"14.9329681\", \"address\": \"ČS Shell - Dálnice D1, Všechlapy (směr Brno)\"}, {\"title\": \"Costa Express - Dobšice \", \"lat\": \"50.1208568\", \"lng\": \"15.2819909\", \"address\": \"ČS Shell - Dálnice D11, Dobšice (směr Hradec Králové)\"}, {\"title\": \"Costa Express - Sv. Kateřina\", \"lat\": \"49.676872\", \"lng\": \"12.5970409\", \"address\": \"ČS Shell - Dálnice D5, Sv. Kateřina (směr Praha)\"}, {\"title\": \"Costa Express - Znojmo\", \"lat\": \"48.85011000000001\", \"lng\": \"16.0544142\", \"address\": \"ČS Shell - Znojmo, ulice Vídeňská 20\"}, {\"title\": \"Costa Express - Plzeň\", \"lat\": \"49.7269928\", \"lng\": \"13.4131866\", \"address\": \"ČS Shell - Plzeň, ulice Koterovská 2474/156\"}, {\"title\": \"Costa Express - Chrudim\", \"lat\": \"49.9553915\", \"lng\": \"15.7992976\", \"address\": \"ČS Shell - Chrudim, ulice Dašická 1228\"}, {\"title\": \"Costa Express - Valašské Meziříčí\", \"lat\": \"49.48404\", \"lng\": \"17.9669745\", \"address\": \"ČS Shell - Valašské Meziříčí, ulice Masarykova 672/35\"}, {\"title\": \"Inmedio Nile House Praha 8\", \"lat\": \"50.09431670000001\", \"lng\": \"14.4451448\", \"address\": \"Inmedio Nile House Praha 8, ulice Karolínská 654/2\"}, {\"title\": \"Inmedio Park Chodov Praha 4\", \"lat\": \"50.02840579999999\", \"lng\": \"14.4934399\", \"address\": \"Inmedio Park Chodov Praha 4, ulice V parku 2308/8\"}, {\"title\": \"Inmedio Classic 7 BP Praha 7,\", \"lat\": \"50.1074563\", \"lng\": \"14.4540885\", \"address\": \"Inmedio Classic 7 BP Praha 7, ulice Jankovcova 1037/49\"}, {\"title\": \"Costa Express - Ostrava\", \"lat\": \"49.7963604\", \"lng\": \"18.2714126\", \"address\": \"ČS Shell - Ostrava, ulice Místecká 1626\"}, {\"title\": \"Costa Express - Nupaky\", \"lat\": \"49.9851061\", \"lng\": \"14.6010623\", \"address\": \"ČS Shell - Dálnice D1 (směr Praha)\"}, {\"title\": \"Costa Express  - Slaný\", \"lat\": \"50.2084898\", \"lng\": \"14.0849532\", \"address\": \"ČS Shell - Rych.Kom. I/7, směr Louny\"}, {\"title\": \"Inmedio Kavčí Hory, OFFICE PARK\", \"lat\": \"50.0517239\", \"lng\": \"14.430839\", \"address\": \"Inmedio Kavčí Hory, OFFICE PARK, Na Hřebenech II/1718, Praha 4\"}, {\"title\": \" Hello! Nám. Republiky, vestibul metra „B“ u vchodu do Palladia\", \"lat\": \"50.08881410000001\", \"lng\": \"14.4288218\", \"address\": \" Hello! Nám. Republiky, vestibul metra „B“ u vchodu do Palladia, Nám. Republiky 1, Praha 1\"}, {\"title\": \"Inmedio Thámova\", \"lat\": \"50.09189569999999\", \"lng\": \"14.4523579\", \"address\": \"Inmedio Thámova, Praha 8, Thámova 11\"}, {\"title\": \"Tesco Letňany\", \"lat\": \"50.1350535\", \"lng\": \"14.5012747\", \"address\": \"Tesco Letňany, Praha 9, Veselská 663\"}, {\"title\": \"Costa Express - Most\", \"lat\": \"50.5164389\", \"lng\": \"13.6397737\", \"address\": \"ČS Shell - Most, ulice Chomutovská 1197\"}, {\"title\": \"Costa Express - Hora Svatého Šebestiána\", \"lat\": \"50.5088523\", \"lng\": \"13.2509598\", \"address\": \"ČS Shell - Hora Svatého Šebestiána 101\"}, {\"title\": \" Costa Express -  1 Minute Lighthouse\", \"lat\": \"50.102849\", \"lng\": \"14.4567696\", \"address\": \"1 Minute Lighthouse, Praha 7 ulice Jankovcova 1569/2c\"}, {\"title\": \"Hello! Hostivař, Depo Hostivař metro „A“, Sázečská\", \"lat\": \"50.07608785721049\", \"lng\": \"14.515799699752847\", \"address\": \"Hello! Hostivař, Depo Hostivař metro „A“, Sázečská\\n\"}, {\"title\": \"ČS Shell -  Karlovy Vary\", \"lat\": \"50.2415116\", \"lng\": \"12.8704748\", \"address\": \"Karlovy Vary, ulice Jáchymovská 445\"}]\n"
     ]
    }
   ],
   "source": [
    "print_json(b)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}