Costa_parse_feed.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib\n",
    "from urllib.error import HTTPError\n",
    "import requests\n",
    "import ssl\n",
    "ssl._create_default_https_context = ssl._create_unverified_context\n",
    "import csv\n",
    "from xml import etree\n",
    "from xml.sax.saxutils import unescape\n",
    "import xml.etree.ElementTree as ET\n",
    "import json\n",
    "import html\n",
    "import pandas as pd\n",
    "from pandas.io.json import json_normalize\n",
    "import numpy as np\n",
    "import unicodedata\n",
    "import time\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "google_base_url = \"https://maps.googleapis.com/maps/api/geocode/json?address=\"\n",
    "auth_token = (env.token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_root(url):\n",
    "    req = urllib.request.Request(url=url)\n",
    "    try:\n",
    "        handler = urllib.request.urlopen(req)\n",
    "        tree = ET.ElementTree(file = handler)\n",
    "        root = tree.getroot()\n",
    "        return root\n",
    "    except HTTPError as e:\n",
    "        content = e.read()\n",
    "        return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = \"https://www.costa-coffee.cz/kavarny.php\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "root = get_root(base_url)                        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_column_names(root):\n",
    "    tags = []\n",
    "    for i, child in enumerate(root):  \n",
    "        for subchild in child[-1:]: \n",
    "            for k in subchild:\n",
    "                tags.append(k.tag)\n",
    "    return tags              "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = get_column_names(root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['title',\n",
       " 'description',\n",
       " 'pubDate',\n",
       " 'category',\n",
       " 'image',\n",
       " 'link',\n",
       " 'guid',\n",
       " 'wifi',\n",
       " 'kuracka_zona',\n",
       " 'bezbarierovy_pristup',\n",
       " 'detsky_koutek',\n",
       " 'oteviraci_doba']"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(root):\n",
    "    records = []\n",
    "    data = []\n",
    "    for i, child in enumerate(root):\n",
    "        for subchild in child:\n",
    "            for j in subchild:\n",
    "                records.append(j.text)\n",
    "            try:\n",
    "                if len(records) != 0:\n",
    "                    data.append(records)\n",
    "                    records = []\n",
    "            except AttributeError as error:\n",
    "                return error\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = get_data(root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "rawDf = pd.DataFrame(records, columns=tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unescape(x):\n",
    "        return (html.unescape(x).encode(\"utf-8\", \"xmlcharrefreplace\").decode())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "unescapeDf = rawDf\n",
    "unescapeDf[\"description\"] = unescapeDf[\"description\"].apply(unescape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# maze nepotrebne sloupce\n",
    "unescapeDf.drop(unescapeDf.columns[[2,3, 5, 6]], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prejmenuje sloupce\n",
    "renameUnescapeDf = unescapeDf.rename(columns={'description': 'address', 'kuracka_zona': 'smoking_zone', 'bezbarierovy_pristup': 'accessible', 'detsky_koutek': 'playground','oteviraci_doba': 'opening_hours'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_request(address, google_base_url, auth_token):\n",
    "        url = google_base_url + urllib.parse.quote(address) + auth_token\n",
    "        try:\n",
    "            response = requests.get(url)\n",
    "            response_json = json.loads(response.text)\n",
    "            return response_json\n",
    "        except response == \"\":\n",
    "            response_json = NaN\n",
    "        except requests.exceptions.HTTPError as errh:\n",
    "            print (\"Http Error:\",errh)\n",
    "        except requests.exceptions.ConnectionError as errc:\n",
    "            print (\"Error Connecting:\",errc)\n",
    "        except requests.exceptions.Timeout as errt:\n",
    "            print (\"Timeout Error:\",errt)\n",
    "        except requests.exceptions.RequestException as err:\n",
    "            print (\"OOps: Something Else\",err)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_coordinates(address):\n",
    "    response = make_request(address, google_base_url, auth_token)\n",
    "    result = {}\n",
    "    lat = str(round(response[\"results\"][0][\"geometry\"][\"location\"][\"lat\"],6))\n",
    "    lng = str(round(response[\"results\"][0][\"geometry\"][\"location\"][\"lng\"],6))\n",
    "    result = {\"lat\":lat, \"lng\":lng}\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sanitize_html(value):\n",
    "    value = value.strip()\n",
    "    value = value.replace('\\n','')\n",
    "    value = value.replace('\\t','')\n",
    "    return re.sub(r'<[^>]*?>', '', value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "def coordinates(row):\n",
    "    address_coordinates = get_coordinates(row[\"address\"])\n",
    "    row[\"lat\"] = address_coordinates[\"lat\"]\n",
    "    row[\"lng\"] = address_coordinates[\"lng\"]\n",
    "    return row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "sanitizedAddressDf = renameUnescapeDf\n",
    "sanitizedAddressDf[\"address\"] = sanitizedAddressDf[\"address\"].apply(sanitize_html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "sanitizedTimesDf = sanitizedAddressDf\n",
    "sanitizedTimesDf[\"opening_hours\"] = sanitizedTimesDf[\"opening_hours\"].apply(sanitize_html)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "coordinatesDf = sanitizedTimesDf\n",
    "coordinatesDf = coordinatesDf.apply(coordinates, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "YesNoDf = coordinatesDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def yes_no_recode(value):\n",
    "    recoded = np.where(value == \"Ano\",1,0)\n",
    "    return recoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "YesNoDf[\"wifi\"] = YesNoDf[\"wifi\"].apply(yes_no_recode)\n",
    "YesNoDf[\"smoking_zone\"] = YesNoDf[\"smoking_zone\"].apply(yes_no_recode)\n",
    "YesNoDf[\"accessible\"] = YesNoDf[\"accessible\"].apply(yes_no_recode)\n",
    "YesNoDf[\"playground\"] = YesNoDf[\"playground\"].apply(yes_no_recode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_to_json(df):\n",
    "    feed_json = df.to_json(orient='records')\n",
    "    temp_po = json.loads(feed_json, encoding='utf-8')\n",
    "    temp_json = json.dumps(temp_po, ensure_ascii = False)\n",
    "    json_no_space = temp_json.replace('\\\\', '')\n",
    "    json_no_space = temp_json.replace('\\n', ' ')\n",
    "    return temp_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "feed_json = df_to_json(YesNoDf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_json(json):\n",
    "    return print(json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_json(feed_json)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}