get-taiwan2020-election-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from selenium import webdriver\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Reference: python 網路爬蟲教學-Selenium基本操作, [even](https://freelancerlife.info/zh/blog/python%E7%B6%B2%E8%B7%AF%E7%88%AC%E8%9F%B2%E6%95%99%E5%AD%B8-selenium%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C/?fbclid=IwAR0IWwLhw-1eIosCInihhDUbvASD7_aNWr17SK-X9XypqpRAssZIb4YsAuw)\n",
    "## 2. download webdriver by your browser version ex. Chrome"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.cec.gov.tw/pc/zh_TW/L4/n00000000000000000.html'\n",
    "driver_path = '/Users/zoe/Desktop/ptt_crawler/github_data/chromedriver'\n",
    "rs_df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this list for exclude\n",
    "cnty_lst = ['政黨得票數 (不分區及僑居國外國民立法委員)',\n",
    " '臺北市',\n",
    " '新北市',\n",
    " '桃園市',\n",
    " '臺中市',\n",
    " '臺南市',\n",
    " '高雄市',\n",
    " '新竹縣',\n",
    " '苗栗縣',\n",
    " '彰化縣',\n",
    " '南投縣',\n",
    " '雲林縣',\n",
    " '嘉義縣',\n",
    " '屏東縣',\n",
    " '宜蘭縣',\n",
    " '花蓮縣',\n",
    " '臺東縣',\n",
    " '澎湖縣',\n",
    " '金門縣',\n",
    " '連江縣',\n",
    " '基隆市',\n",
    " '新竹市',\n",
    " '嘉義市'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "county: 臺北市 starts!\n",
      "松山區\n",
      "信義區\n",
      "大安區\n",
      "中山區\n",
      "中正區\n",
      "大同區\n",
      "萬華區\n",
      "文山區\n",
      "南港區\n",
      "內湖區\n",
      "士林區\n",
      "北投區\n",
      "county: 新北市 starts!\n",
      "板橋區\n",
      "三重區\n",
      "中和區\n",
      "永和區\n",
      "新莊區\n",
      "新店區\n",
      "樹林區\n",
      "鶯歌區\n",
      "三峽區\n",
      "淡水區\n",
      "汐止區\n",
      "瑞芳區\n",
      "土城區\n",
      "蘆洲區\n",
      "五股區\n",
      "泰山區\n",
      "林口區\n",
      "深坑區\n",
      "石碇區\n",
      "坪林區\n",
      "三芝區\n",
      "石門區\n",
      "八里區\n",
      "平溪區\n",
      "雙溪區\n",
      "貢寮區\n",
      "金山區\n",
      "萬里區\n",
      "烏來區\n",
      "county: 桃園市 starts!\n",
      "桃園區\n",
      "中壢區\n",
      "大溪區\n",
      "楊梅區\n",
      "蘆竹區\n",
      "大園區\n",
      "龜山區\n",
      "八德區\n",
      "龍潭區\n",
      "平鎮區\n",
      "新屋區\n",
      "觀音區\n",
      "復興區\n",
      "county: 臺中市 starts!\n",
      "中區\n",
      "東區\n",
      "南區\n",
      "西區\n",
      "北區\n",
      "西屯區\n",
      "南屯區\n",
      "北屯區\n",
      "豐原區\n",
      "東勢區\n",
      "大甲區\n",
      "清水區\n",
      "沙鹿區\n",
      "梧棲區\n",
      "后里區\n",
      "神岡區\n",
      "潭子區\n",
      "大雅區\n",
      "新社區\n",
      "石岡區\n",
      "外埔區\n",
      "大安區\n",
      "烏日區\n",
      "大肚區\n",
      "龍井區\n",
      "霧峰區\n",
      "太平區\n",
      "大里區\n",
      "和平區\n",
      "county: 臺南市 starts!\n",
      "新營區\n",
      "鹽水區\n",
      "白河區\n",
      "柳營區\n",
      "後壁區\n",
      "東山區\n",
      "麻豆區\n",
      "下營區\n",
      "六甲區\n",
      "官田區\n",
      "大內區\n",
      "佳里區\n",
      "學甲區\n",
      "西港區\n",
      "七股區\n",
      "將軍區\n",
      "北門區\n",
      "新化區\n",
      "善化區\n",
      "新市區\n",
      "安定區\n",
      "山上區\n",
      "玉井區\n",
      "楠西區\n",
      "南化區\n",
      "左鎮區\n",
      "仁德區\n",
      "歸仁區\n",
      "關廟區\n",
      "龍崎區\n",
      "永康區\n",
      "東區\n",
      "南區\n",
      "北區\n",
      "安南區\n",
      "安平區\n",
      "中西區\n",
      "county: 高雄市 starts!\n",
      "鹽埕區\n",
      "鼓山區\n",
      "左營區\n",
      "楠梓區\n",
      "三民區\n",
      "新興區\n",
      "前金區\n",
      "苓雅區\n",
      "前鎮區\n",
      "旗津區\n",
      "小港區\n",
      "鳳山區\n",
      "林園區\n",
      "大寮區\n",
      "大樹區\n",
      "大社區\n",
      "仁武區\n",
      "鳥松區\n",
      "岡山區\n",
      "橋頭區\n",
      "燕巢區\n",
      "田寮區\n",
      "阿蓮區\n",
      "路竹區\n",
      "湖內區\n",
      "茄萣區\n",
      "永安區\n",
      "彌陀區\n",
      "梓官區\n",
      "旗山區\n",
      "美濃區\n",
      "六龜區\n",
      "甲仙區\n",
      "杉林區\n",
      "內門區\n",
      "茂林區\n",
      "桃源區\n",
      "那瑪夏區\n",
      "county: 新竹縣 starts!\n",
      "竹北市\n",
      "竹東鎮\n",
      "新埔鎮\n",
      "關西鎮\n",
      "湖口鄉\n",
      "新豐鄉\n",
      "芎林鄉\n",
      "橫山鄉\n",
      "北埔鄉\n",
      "寶山鄉\n",
      "峨眉鄉\n",
      "尖石鄉\n",
      "五峰鄉\n",
      "county: 苗栗縣 starts!\n",
      "苗栗市\n",
      "苑裡鎮\n",
      "通霄鎮\n",
      "竹南鎮\n",
      "頭份市\n",
      "後龍鎮\n",
      "卓蘭鎮\n",
      "大湖鄉\n",
      "公館鄉\n",
      "銅鑼鄉\n",
      "南庄鄉\n",
      "頭屋鄉\n",
      "三義鄉\n",
      "西湖鄉\n",
      "造橋鄉\n",
      "三灣鄉\n",
      "獅潭鄉\n",
      "泰安鄉\n",
      "county: 彰化縣 starts!\n",
      "彰化市\n",
      "鹿港鎮\n",
      "和美鎮\n",
      "線西鄉\n",
      "伸港鄉\n",
      "福興鄉\n",
      "秀水鄉\n",
      "花壇鄉\n",
      "芬園鄉\n",
      "員林市\n",
      "溪湖鎮\n",
      "田中鎮\n",
      "大村鄉\n",
      "埔鹽鄉\n",
      "埔心鄉\n",
      "永靖鄉\n",
      "社頭鄉\n",
      "二水鄉\n",
      "北斗鎮\n",
      "二林鎮\n",
      "田尾鄉\n",
      "埤頭鄉\n",
      "芳苑鄉\n",
      "大城鄉\n",
      "竹塘鄉\n",
      "溪州鄉\n",
      "county: 南投縣 starts!\n",
      "南投市\n",
      "埔里鎮\n",
      "草屯鎮\n",
      "竹山鎮\n",
      "集集鎮\n",
      "名間鄉\n",
      "鹿谷鄉\n",
      "中寮鄉\n",
      "魚池鄉\n",
      "國姓鄉\n",
      "水里鄉\n",
      "信義鄉\n",
      "仁愛鄉\n",
      "county: 雲林縣 starts!\n",
      "斗六市\n",
      "斗南鎮\n",
      "虎尾鎮\n",
      "西螺鎮\n",
      "土庫鎮\n",
      "北港鎮\n",
      "古坑鄉\n",
      "大埤鄉\n",
      "莿桐鄉\n",
      "林內鄉\n",
      "二崙鄉\n",
      "崙背鄉\n",
      "麥寮鄉\n",
      "東勢鄉\n",
      "褒忠鄉\n",
      "臺西鄉\n",
      "元長鄉\n",
      "四湖鄉\n",
      "口湖鄉\n",
      "水林鄉\n",
      "county: 嘉義縣 starts!\n",
      "太保市\n",
      "朴子市\n",
      "布袋鎮\n",
      "大林鎮\n",
      "民雄鄉\n",
      "溪口鄉\n",
      "新港鄉\n",
      "六腳鄉\n",
      "東石鄉\n",
      "義竹鄉\n",
      "鹿草鄉\n",
      "水上鄉\n",
      "中埔鄉\n",
      "竹崎鄉\n",
      "梅山鄉\n",
      "番路鄉\n",
      "大埔鄉\n",
      "阿里山鄉\n",
      "county: 屏東縣 starts!\n",
      "屏東市\n",
      "潮州鎮\n",
      "東港鎮\n",
      "恆春鎮\n",
      "萬丹鄉\n",
      "長治鄉\n",
      "麟洛鄉\n",
      "九如鄉\n",
      "里港鄉\n",
      "鹽埔鄉\n",
      "高樹鄉\n",
      "萬巒鄉\n",
      "內埔鄉\n",
      "竹田鄉\n",
      "新埤鄉\n",
      "枋寮鄉\n",
      "新園鄉\n",
      "崁頂鄉\n",
      "林邊鄉\n",
      "南州鄉\n",
      "佳冬鄉\n",
      "琉球鄉\n",
      "車城鄉\n",
      "滿州鄉\n",
      "枋山鄉\n",
      "三地門鄉\n",
      "霧臺鄉\n",
      "瑪家鄉\n",
      "泰武鄉\n",
      "來義鄉\n",
      "春日鄉\n",
      "獅子鄉\n",
      "牡丹鄉\n",
      "county: 宜蘭縣 starts!\n",
      "宜蘭市\n",
      "羅東鎮\n",
      "蘇澳鎮\n",
      "頭城鎮\n",
      "礁溪鄉\n",
      "壯圍鄉\n",
      "員山鄉\n",
      "冬山鄉\n",
      "五結鄉\n",
      "三星鄉\n",
      "大同鄉\n",
      "南澳鄉\n",
      "county: 花蓮縣 starts!\n",
      "花蓮市\n",
      "鳳林鎮\n",
      "玉里鎮\n",
      "新城鄉\n",
      "吉安鄉\n",
      "壽豐鄉\n",
      "光復鄉\n",
      "豐濱鄉\n",
      "瑞穗鄉\n",
      "富里鄉\n",
      "秀林鄉\n",
      "萬榮鄉\n",
      "卓溪鄉\n",
      "county: 臺東縣 starts!\n",
      "臺東市\n",
      "成功鎮\n",
      "關山鎮\n",
      "卑南鄉\n",
      "鹿野鄉\n",
      "池上鄉\n",
      "東河鄉\n",
      "長濱鄉\n",
      "太麻里鄉\n",
      "大武鄉\n",
      "綠島鄉\n",
      "海端鄉\n",
      "延平鄉\n",
      "金峰鄉\n",
      "達仁鄉\n",
      "蘭嶼鄉\n",
      "county: 澎湖縣 starts!\n",
      "馬公市\n",
      "湖西鄉\n",
      "白沙鄉\n",
      "西嶼鄉\n",
      "望安鄉\n",
      "七美鄉\n",
      "county: 金門縣 starts!\n",
      "金城鎮\n",
      "金沙鎮\n",
      "金湖鎮\n",
      "金寧鄉\n",
      "烈嶼鄉\n",
      "烏坵鄉\n",
      "county: 連江縣 starts!\n",
      "南竿鄉\n",
      "北竿鄉\n",
      "莒光鄉\n",
      "東引鄉\n",
      "county: 基隆市 starts!\n",
      "中正區\n",
      "七堵區\n",
      "暖暖區\n",
      "仁愛區\n",
      "中山區\n",
      "安樂區\n",
      "信義區\n",
      "county: 新竹市 starts!\n",
      "東區\n",
      "北區\n",
      "香山區\n",
      "county: 嘉義市 starts!\n",
      "東區\n",
      "西區\n"
     ]
    }
   ],
   "source": [
    "driver = webdriver.Chrome(driver_path)\n",
    "driver.implicitly_wait(10)\n",
    "driver.get(url)\n",
    "driver.find_element_by_id(\"itemTextLink1260\").click()\n",
    "\n",
    "html = driver.page_source\n",
    "cnty_soup = BeautifulSoup(html)\n",
    "cnty_urls = [ [x['href'][2:], x.text, x['id']] \n",
    "              for x in cnty_soup.find_all(href=re.compile('L4'))[1:]]\n",
    "\n",
    "for cnty in cnty_urls:\n",
    "    print('county: {} starts!'.format(cnty[1]))\n",
    "    driver.find_element_by_id(cnty[2]).click()\n",
    "    html = driver.page_source\n",
    "    city_soup = BeautifulSoup(html)\n",
    "    dist_urls = [ [x['href'][2:], x.text, x['id']] \n",
    "                  for x in city_soup.find_all(href=re.compile('L4')) \n",
    "                  if x.text not in cnty_lst]\n",
    "    for dist in dist_urls:\n",
    "        print(dist[1])\n",
    "        driver.find_element_by_id(dist[2]).click()\n",
    "        html = driver.page_source\n",
    "        dist_soup = BeautifulSoup(html)\n",
    "        temp_df = pd.read_html(str(dist_soup.select('.tableT')[0]))[0]\n",
    "        temp_df['cnty'] = cnty[1]\n",
    "        temp_df['dist'] = dist[1]\n",
    "        rs_df = rs_df.append(temp_df)\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs_df.reset_index(drop=True, inplace=True)\n",
    "rs_df_t1 = rs_df[rs_df[2].str.isdigit()][[1, 2, 3, 'cnty', 'dist']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs_df_t1.columns = ['政黨', '得票數', '得票率', '縣市', '區']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs_df_t1.to_csv('ele_2020.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}