jupyter/Lab2-fetch-data-parse-with-bs4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lab2 目標：抓取大安區公所的逐三小時預報\n",
    "### 有些欄位有 colspan = ? 必須額外展開才能塞進 dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "url = 'http://www.cwb.gov.tw/V7/forecast/town368/3Hr/6300300.htm'\n",
    "req = requests.get(url)\n",
    "req.encoding = 'utf-8' # 不指定會發生編碼錯誤"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "req.status_code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 如果要用類似 scrapy shell debug 的功能，uncomment 下方的 code block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#from scrapy.http import TextResponse\n",
    "#response = TextResponse(req.url, body=req.text, encoding='utf-8') # 變數特別命名的跟 scrapy shell 開起來一樣\n",
    "#soup = BeautifulSoup(response.text, 'html.parser')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用 requests 得到的 html 內容，用 bs4 parse 所需資訊"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(req.text, 'html.parser')\n",
    "trs = soup.find_all('tr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "columns = ['record_t', # 0\n",
    "           'weekday', # 1\n",
    "           'wx', # 2\n",
    "           't', # 3 \n",
    "           'at', # 4\n",
    "           'beaufort', # 5\n",
    "           'wind_dir', # 6\n",
    "           'rh', # 7\n",
    "           'pop', #8 \n",
    "           'ci'] # 9\n",
    "df = pd.DataFrame(columns=columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 以下的 len() == 3 ，先整理 48 小時內的 timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# day repeat time store in colspans\n",
    "# date store in dates\n",
    "# days store in days\n",
    "# len(colspans) == len(dates) == len(days) == 2 or 3\n",
    "import re\n",
    "import datetime\n",
    "year_s = []\n",
    "year_s.append(\"%d\" % datetime.datetime.now().year)\n",
    "year_s.append(\"%d\" % (datetime.datetime.now() + datetime.timedelta(days=1)).year) # in case tmrr is next year\n",
    "year_s.append(\"%d\" % (datetime.datetime.now() + datetime.timedelta(days=2)).year) # in case it's next year in 2 days\n",
    "colspans = []\n",
    "dates = []\n",
    "days = []\n",
    "k = 0\n",
    "for idx, td in enumerate(trs[0].findAll('td')): # trs[0] 是時間相關的列\n",
    "    if idx > 0:\n",
    "        if td.has_attr('colspan'):\n",
    "            colspans.append(td.attrs['colspan'])\n",
    "        else:\n",
    "            colspans.append(\"1\")\n",
    "        days.append(re.findall('[一|二|三|四|五|六|日]', td.text)[0]) # 用正規表示式把星期\"幾\"選出來\n",
    "        month_n_date = re.findall('\\d+', td.text)\n",
    "        dates.append(year_s[k] + '-' + month_n_date[0] + '-' + month_n_date[1])\n",
    "        k+=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 以下的 len() == 17，行數=17，列數不含 index = 10\n",
    "### 範例\n",
    "\n",
    "| pk | record_t(0) | weekday(1) | wx(2) | t(3) | at(4) | beaufort(5) | wind_dir(6) | rh(7) | pop(8) | ci(9) | \n",
    "| :--: | :--: | :--: | :--: | :--: | :--: | :--: | :--: | :--: | :--: | :--: |\n",
    "| 1 | 2017-08-15 09:00 | 星期二 | 晴天 | 33 | 38 | 2 | 偏西風 | 78% | 0% | 悶熱"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 處理第 0:1 行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "record_ts = []\n",
    "weekdays = []\n",
    "hours = trs[1].findAll('span')\n",
    "k = 0\n",
    "for i in range(0, len(colspans)):\n",
    "    for j in range(0, int(colspans[i])):\n",
    "        record_ts.append(dates[i] + 'T' + hours[k].text)\n",
    "        k+=1\n",
    "        weekdays.append(days[i])\n",
    "\n",
    "df['record_t'] = record_ts\n",
    "df['weekday'] = weekdays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 處理第 2 行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "wxs = []\n",
    "for img in trs[2].findAll('img'):\n",
    "    wxs.append(img.attrs['alt'])\n",
    "\n",
    "df['wx'] = wxs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 處理第 3:9 行，除了第 8 行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vals = []\n",
    "for i in range(3, 10):\n",
    "    if i is not 8: # 降雨機率可能要展開 colspan == 2\n",
    "        tds = trs[i].findAll('td')\n",
    "        for idx, td in enumerate(tds):\n",
    "            if idx > 0:\n",
    "                vals.append(td.text)\n",
    "        df.iloc[:,i] = vals\n",
    "        vals = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 處理第 8 行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pops = [] # probability of precipitation\n",
    "rep = 0\n",
    "for idx, td in enumerate(trs[8].findAll('td')):\n",
    "    if idx > 0:\n",
    "        if td.has_attr('colspan'):\n",
    "            rep = int(td.attrs['colspan'])\n",
    "        else:\n",
    "            rep = 1\n",
    "        for i in range(0, rep):\n",
    "            pops.append(td.text)\n",
    "\n",
    "df['pop'] = pops"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 呈現"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record_t</th>\n",
       "      <th>weekday</th>\n",
       "      <th>wx</th>\n",
       "      <th>t</th>\n",
       "      <th>at</th>\n",
       "      <th>beaufort</th>\n",
       "      <th>wind_dir</th>\n",
       "      <th>rh</th>\n",
       "      <th>pop</th>\n",
       "      <th>ci</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-08-24 18:00</td>\n",
       "      <td>四</td>\n",
       "      <td>多雲</td>\n",
       "      <td>32</td>\n",
       "      <td>37</td>\n",
       "      <td>2</td>\n",
       "      <td>西北風</td>\n",
       "      <td>80%</td>\n",
       "      <td>10%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-08-24 21:00</td>\n",
       "      <td>四</td>\n",
       "      <td>多雲</td>\n",
       "      <td>31</td>\n",
       "      <td>36</td>\n",
       "      <td>&lt;=1</td>\n",
       "      <td>偏南風</td>\n",
       "      <td>85%</td>\n",
       "      <td>10%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-08-25 00:00</td>\n",
       "      <td>五</td>\n",
       "      <td>多雲</td>\n",
       "      <td>30</td>\n",
       "      <td>35</td>\n",
       "      <td>&lt;=1</td>\n",
       "      <td>西北風</td>\n",
       "      <td>89%</td>\n",
       "      <td>20%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-08-25 03:00</td>\n",
       "      <td>五</td>\n",
       "      <td>多雲</td>\n",
       "      <td>29</td>\n",
       "      <td>35</td>\n",
       "      <td>&lt;=1</td>\n",
       "      <td>偏南風</td>\n",
       "      <td>92%</td>\n",
       "      <td>20%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-08-25 06:00</td>\n",
       "      <td>五</td>\n",
       "      <td>晴天</td>\n",
       "      <td>28</td>\n",
       "      <td>33</td>\n",
       "      <td>&lt;=1</td>\n",
       "      <td>偏東風</td>\n",
       "      <td>88%</td>\n",
       "      <td>20%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2017-08-25 09:00</td>\n",
       "      <td>五</td>\n",
       "      <td>晴天</td>\n",
       "      <td>32</td>\n",
       "      <td>36</td>\n",
       "      <td>2</td>\n",
       "      <td>偏東風</td>\n",
       "      <td>68%</td>\n",
       "      <td>20%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2017-08-25 12:00</td>\n",
       "      <td>五</td>\n",
       "      <td>午後短暫雷陣雨</td>\n",
       "      <td>35</td>\n",
       "      <td>42</td>\n",
       "      <td>2</td>\n",
       "      <td>偏東風</td>\n",
       "      <td>80%</td>\n",
       "      <td>30%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2017-08-25 15:00</td>\n",
       "      <td>五</td>\n",
       "      <td>午後短暫雷陣雨</td>\n",
       "      <td>34</td>\n",
       "      <td>41</td>\n",
       "      <td>&lt;=1</td>\n",
       "      <td>偏東風</td>\n",
       "      <td>80%</td>\n",
       "      <td>30%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2017-08-25 18:00</td>\n",
       "      <td>五</td>\n",
       "      <td>晴天</td>\n",
       "      <td>32</td>\n",
       "      <td>37</td>\n",
       "      <td>&lt;=1</td>\n",
       "      <td>偏南風</td>\n",
       "      <td>72%</td>\n",
       "      <td>10%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2017-08-25 21:00</td>\n",
       "      <td>五</td>\n",
       "      <td>多雲</td>\n",
       "      <td>31</td>\n",
       "      <td>36</td>\n",
       "      <td>&lt;=1</td>\n",
       "      <td>西南風</td>\n",
       "      <td>87%</td>\n",
       "      <td>10%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2017-08-26 00:00</td>\n",
       "      <td>六</td>\n",
       "      <td>陰天</td>\n",
       "      <td>30</td>\n",
       "      <td>35</td>\n",
       "      <td>&lt;=1</td>\n",
       "      <td>東南風</td>\n",
       "      <td>93%</td>\n",
       "      <td>10%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2017-08-26 03:00</td>\n",
       "      <td>六</td>\n",
       "      <td>陰天</td>\n",
       "      <td>29</td>\n",
       "      <td>35</td>\n",
       "      <td>&lt;=1</td>\n",
       "      <td>西北風</td>\n",
       "      <td>94%</td>\n",
       "      <td>10%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2017-08-26 06:00</td>\n",
       "      <td>六</td>\n",
       "      <td>陰天</td>\n",
       "      <td>28</td>\n",
       "      <td>33</td>\n",
       "      <td>&lt;=1</td>\n",
       "      <td>西南風</td>\n",
       "      <td>88%</td>\n",
       "      <td>0%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2017-08-26 09:00</td>\n",
       "      <td>六</td>\n",
       "      <td>陰天</td>\n",
       "      <td>32</td>\n",
       "      <td>36</td>\n",
       "      <td>2</td>\n",
       "      <td>偏北風</td>\n",
       "      <td>67%</td>\n",
       "      <td>0%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2017-08-26 12:00</td>\n",
       "      <td>六</td>\n",
       "      <td>午後短暫雷陣雨</td>\n",
       "      <td>35</td>\n",
       "      <td>42</td>\n",
       "      <td>2</td>\n",
       "      <td>西北風</td>\n",
       "      <td>80%</td>\n",
       "      <td>30%</td>\n",
       "      <td>易中暑</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2017-08-26 15:00</td>\n",
       "      <td>六</td>\n",
       "      <td>午後短暫雷陣雨</td>\n",
       "      <td>34</td>\n",
       "      <td>40</td>\n",
       "      <td>2</td>\n",
       "      <td>西南風</td>\n",
       "      <td>80%</td>\n",
       "      <td>30%</td>\n",
       "      <td>易中暑</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2017-08-26 18:00</td>\n",
       "      <td>六</td>\n",
       "      <td>多雲</td>\n",
       "      <td>32</td>\n",
       "      <td>37</td>\n",
       "      <td>&lt;=1</td>\n",
       "      <td>東南風</td>\n",
       "      <td>73%</td>\n",
       "      <td>0%</td>\n",
       "      <td>悶熱</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            record_t weekday       wx   t  at beaufort wind_dir   rh  pop   ci\n",
       "0   2017-08-24 18:00       四       多雲  32  37        2      西北風  80%  10%   悶熱\n",
       "1   2017-08-24 21:00       四       多雲  31  36      <=1      偏南風  85%  10%   悶熱\n",
       "2   2017-08-25 00:00       五       多雲  30  35      <=1      西北風  89%  20%   悶熱\n",
       "3   2017-08-25 03:00       五       多雲  29  35      <=1      偏南風  92%  20%   悶熱\n",
       "4   2017-08-25 06:00       五       晴天  28  33      <=1      偏東風  88%  20%   悶熱\n",
       "5   2017-08-25 09:00       五       晴天  32  36        2      偏東風  68%  20%   悶熱\n",
       "6   2017-08-25 12:00       五  午後短暫雷陣雨  35  42        2      偏東風  80%  30%   悶熱\n",
       "7   2017-08-25 15:00       五  午後短暫雷陣雨  34  41      <=1      偏東風  80%  30%   悶熱\n",
       "8   2017-08-25 18:00       五       晴天  32  37      <=1      偏南風  72%  10%   悶熱\n",
       "9   2017-08-25 21:00       五       多雲  31  36      <=1      西南風  87%  10%   悶熱\n",
       "10  2017-08-26 00:00       六       陰天  30  35      <=1      東南風  93%  10%   悶熱\n",
       "11  2017-08-26 03:00       六       陰天  29  35      <=1      西北風  94%  10%   悶熱\n",
       "12  2017-08-26 06:00       六       陰天  28  33      <=1      西南風  88%   0%   悶熱\n",
       "13  2017-08-26 09:00       六       陰天  32  36        2      偏北風  67%   0%   悶熱\n",
       "14  2017-08-26 12:00       六  午後短暫雷陣雨  35  42        2      西北風  80%  30%  易中暑\n",
       "15  2017-08-26 15:00       六  午後短暫雷陣雨  34  40        2      西南風  80%  30%  易中暑\n",
       "16  2017-08-26 18:00       六       多雲  32  37      <=1      東南風  73%   0%   悶熱"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 存檔"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out_csv = './csv/daan_3hr.csv'\n",
    "df.to_csv(out_csv, sep=',', encoding='utf-8', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}