CRAWLING_FIRE_CENTER_DATA_20180213_VER7_FireFox.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Crawling National Fire Stastistics from NFDS(National Fire Data System) by manipulating Javascript using Selenium and PhantomJS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import Select\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "import datetime\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#This crawler tested Chrome, PhantomJS(headless browser), and Firefox to implement crawler\n",
    "path_chrome = r\"C:\\Users\\Administrator\\Anaconda3\\chromedriver_win32\\chromedriver.exe\"\n",
    "path_js = r\"C:\\Users\\Administrator\\Anaconda3\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe\"\n",
    "path_gecko = r\"C:/Users/Administrator/Anaconda3/geckodriver-v0.19.1-win64/geckodriver.exe\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument('--headless')\n",
    "options.add_argument('--no-sandbox')\n",
    "options.add_argument('--disable-gpu')\n",
    "chrome = webdriver.Chrome(executable_path=path_chrome, chrome_options=options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "firefox = webdriver.Firefox(executable_path=path_gecko)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "js = webdriver.PhantomJS(path_js)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#URL of fire data categorized by Province\n",
    "url_area= \"http://nfds.go.kr/fr_base_0001.jsf\"\n",
    "table_selector_area = '#iTable > table > tbody > tr'\n",
    "xpath_area ='//*[@id=\"content-area\"]/div[2]/div[2]/form/table/tbody/tr/td[2]/select[1]'\n",
    "columns_area=['구분','계','실화_건수','자연적요인_건수','방화_건수','미상_건수','실화_비율',\n",
    "'자연적요인_비율','방화_비율','미상_비율','인명피해_계','인명피해_사망','인명피해_부상',\n",
    "'재산피해_계','재산피해_부동산','재산피해_동산','소실면석_동수','소실면적_면적','도','날짜']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#URL of fire data categorized by Fire Station\n",
    "url_station =\"http://nfds.go.kr/fr_base_0101.jsf\"\n",
    "table_selector_station = '#iTable > table > tbody > tr'\n",
    "xpath_station_1 = '//*[@id=\"slFireCenter1\"]'\n",
    "columns_station=['구분','계','실화_건','자연적요인_건','방화_건','미상_건','실화_비율',\n",
    "                '자연적요인_비율','방화_비율','미상_비율','인명피해_계','인명피해_사망',\n",
    "                '인명피해_부상','재산피해_계','재산피해_부동산','재산피해_동산',\n",
    "                '소실면적_동수','소실면적_면적','본부','날짜']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#URL of fire data categorized by Fire Prevention System\n",
    "url_facility = 'http://nfds.go.kr/fr_pos_0001.jsf'\n",
    "#When parsing the tables Table, I skipped the first row because it contained useless content\n",
    "table_selector_facility_1 = '#iTable > table > tbody > tr > td:nth-of-type(1) > table > tbody > tr'\n",
    "table_selector_facility_2 = '#tbHeadDiv > table > tbody > tr'\n",
    "type_xpath_facility='//*[@id=\"content-area\"]/div[2]/div[2]/form/table/tbody/tr[1]/td[2]/select'\n",
    "xpath_facility_1='//*[@id=\"slFireCenter1\"]'\n",
    "xpath_facility_2='//*[@id=\"slFireCenter2\"]'\n",
    "columns_facility =['구분_대','구분_소','합계','소화기구_계','소화기구_사용','소화기구_미사용',\n",
    "                 '소화기구_미상','옥내소화전_계','옥내소화전_사용','옥내소화전_미사용/효과미비','옥내소화전_미상',\n",
    "                 '스프링클러등_계','스프링쿨러등_효과적작용','스프링쿨러등_소규모화재로미작동','스프링쿨러등_미작동/효과없음',\n",
    "                 '비상경보설비_계','비상경보설비_경보','비상경보설비_미경보','비상경보설비_미상',\n",
    "                 '자동화재탐지설비_계','자동화재탐지설비_작동','자동화재탐지설비_소규모화재로미작동','자동화재탐지설비_미상',\n",
    "                 '방화문_계','방화문_정상','방화문_비정상','방화구획_계','방화구획_설치','방화구획_미설치','데이터종류','본부','소방서','날짜']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#URL of fire data categorized by PUBLICLY USED ESTABLISHMENTS\n",
    "url_multi = \"http://nfds.go.kr/fr_pos_0201.jsf\"\n",
    "table_selector_multi = '#iTable > table > tbody > tr'\n",
    "type_xpath_multi='//*[@id=\"content-area\"]/div[2]/div[2]/form/table/tbody/tr[1]/td[2]/select'\n",
    "xpath_multi_1='//*[@id=\"slFireCenter1\"]'\n",
    "xpath_multi_2='//*[@id=\"slFireCenter2\"]'\n",
    "columns_multi=['구분_대','구분_소','계','전기적','기계적','화학적','가스누출','교통사고',\n",
    "             '부주의','기타','자연적요인','방화','방화의심','미상','데이터종류','본부','소방서','날짜']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#URL of fire data categorized by building structure\n",
    "url_bldg=\"http://nfds.go.kr/fr_pos_0401.jsf\"\n",
    "table_selector_bldg = '#iTable > table > tbody > tr'\n",
    "type_xpath_bldg = '//*[@id=\"content-area\"]/div[2]/div[2]/form/table/tbody/tr[1]/td[2]/select'\n",
    "xpath_bldg_1 = '//*[@id=\"slFireCenter1\"]'\n",
    "xpath_bldg_2 = '//*[@id=\"slFireCenter2\"]'\n",
    "columns_bldg = ['구분','계','전기적','기계적','화학적요인','가스누출','교통사고','부주의',\n",
    "              '기타','자연적요인','방화','방화의심','미상','구분','본부','소방서','날짜']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#URL of fire data categorized by cause of fire\n",
    "url_rsn_lc = 'http://nfds.go.kr/fr_fact_0001.jsf'\n",
    "table_selector_rsn_lc = '#iTable > table > tbody > tr'\n",
    "type_xpath_rsn_lc = '//*[@id=\"content-area\"]/div[2]/div[2]/form/table/tbody/tr[1]/td[2]/select'\n",
    "xpath_rsn_lc_1 = '//*[@id=\"slFireCenter1\"]'\n",
    "xpath_rsn_lc_2 = '//*[@id=\"slFireCenter2\"]'\n",
    "columns_rsn_lc=['구분_대','구분_소','계','전기적','기계적','화학적요인','가스누출','교통사고',\n",
    "              '부주의','기타','자연적요인','방화','방화의심','미상','구분','본부','소방서','날짜']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#URL of fire data categorized by fire spread cause\n",
    "url_aggv_rsn = \"http://nfds.go.kr/fr_pos_0101.jsf\"\n",
    "table_selector_aggv_rsn = '#iTable > table > tbody > tr'\n",
    "type_xpath_aggv_rsn = '//*[@id=\"content-area\"]/div[2]/div[2]/form/table/tbody/tr[1]/td[2]/select'\n",
    "xpath_aggv_rsn_1 = '//*[@id=\"slFireCenter1\"]'\n",
    "xpath_aggv_rsn_2 = '//*[@id=\"slFireCenter2\"]'\n",
    "columns_aggv_rsn=['구분_대','구분_소','계','전기적','화학적요인','가스누출','교통사고','부주의',\n",
    "                '기타','자연적요인','방화','방화의심','미상','구분','본부','소방서','날짜']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "center_name=['서울소방재난본부','부산소방안전본부','대구소방안전본부','인천소방안전본부','광주소방본부',\n",
    "            '대전소방본부','울산소방본부','세종소방본부','경기재난안전본부','경기북부소방재난본부','강원소방본부','충북소방본부',\n",
    "            '충청남도소방본부','전북소방본부','전남소방본부','경북소방본부','경남소방본부','창원소방본부','제주소방본부']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "station_name=[['종로소방서','중부소방서','광진소방서','용산소방서','동대문소방서','영등포소방서','성북소방서','은평소방서','강남소방서','서초소방서','강서소방서','강동소방서','마포소방서','도봉소방서','구로소방서','노원소방서','관악소방서','송파소방서','양천소방서','중랑소방서','동작소방서','서대문소방서','강북소방서','성동소방서'],\n",
    "             ['중부소방서','부산진소방서','동래소방서','북부소방서','사하소방서','해운대소방서','금정소방서','남부소방서','강서소방서','항만소방서','기장소방서'],\n",
    "             ['중부소방서','동부소방서','서부소방서','북부소방서','수성소방서','달서소방서','달성소방서','강서소방서'],\n",
    "             ['중부소방서','남동소방서','부평소방서','서부소방서','공단소방서','계양소방서','남부소방서','강화소방서','공항소방서','송도소방서'],\n",
    "             ['광산소방서','동부소방서','서부소방서','북부소방서','남부소방서'],\n",
    "             ['중부소방서','서부소방서','동부소방서','북부소방서','남부소방서'],\n",
    "             ['중부소방서','남부소방서','동부소방서','온산소방서'],\n",
    "             ['조치원소방서','세종소방서'],\n",
    "             ['수원소방서','성남소방서','분당소방서','부천소방서','안양소방서','안산소방서','용인소방서','평택소방서','송탄소방서','광명소방서','시흥소방서','군포소방서','화성소방서','이천소방서','김포소방서','광주소방서','안성소방서','하남소방서','의왕소방서','오산소방서','여주소방서','양평소방서','과천소방서','고양소방서','일산소방서','의정부소방서','남양주소방서','파주소방서','구리소방서','포천소방서','양주소방서','동두천소방서','가평소방서','연천소방서'],\n",
    "             ['고양소방서','일산소방서','의정부소방서','남양주소방서','파주소방서','구리소방서','포천소방서','양주소방서','동두천소방서','가평소방서','연천소방서'],\n",
    "              ['춘천소방서','원주소방서','강릉소방서','동해소방서','태백소방서','속초소방서','삼척소방서','홍천소방서','횡성소방서','영월소방서','평창소방서','정선소방서','철원소방서','인제소방서','고성소방서','양양소방서'],\n",
    "              ['청주동부소방서','서부소방서','충주소방서','제천소방서','영동소방서','증평소방서','음성소방서','진천소방서','옥천소방서','보은소방서','괴산소방서','단양소방서'],\n",
    "              ['천안동남소방서','천안서북소방서','공주소방서','보령소방서','아산소방서','서산소방서','논산소방서','당진소방서','금산소방서','부여소방서','서천소방서','홍성소방서','예산소방서','태안소방서','계룡소방서','청양소방서'],\n",
    "              ['전주덕진소방서','전주완산소방서','군산소방서','익산소방서','정읍소방서','남원소방서','김제소방서','고창소방서','부안소방서','무진장소방서'],\n",
    "              ['목포소방서','여수소방서','순천소방서','나주소방서','광양소방서','담양소방서','보성소방서','해남소방서','영암소방서','영광소방서','화순소방서','강진소방서','무안소방서','고흥소방서'],\n",
    "              ['포항북부소방서','포항남부소방서','경주소방서','김천소방서','안동소방서','구미소방서','영주소방서','영천소방서','상주소방서','문경소방서','경산소방서','의성소방서','영덕소방서','청도소방서','고령소방서','성주소방서','칠곡소방서','울진소방서'],\n",
    "              ['진주소방서','통영소방서','사천소방서','밀양소방서','거제소방서','양산소방서','의령소방서','함안소방서','창녕소방서','고성소방서','남해소방서','하동소방서','산청소방서','함양소방서','거창소방서','합천소방서','김해동부소방서','김해서부소방서'],\n",
    "              ['창원소방본부','창원소방서','마산소방서'],\n",
    "              ['제주소방서','서귀포소방서','서부소방서','동부소방서']\n",
    "             ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "station_value=[['1103000,1104000','1105000','1106000',\"1107000\",\"1108000\",'1109000','1112000','1114000','1115000','1117000','1118000','1113000','1110000','1119000','1111000','1116000','1120000','1121000','1122000','1102000','1123000','1125000','1127000'],\n",
    "               ['2601000','2602000','2603000','2604000','2605000','2606000','2607000','2608000','2609000','2610000','2611000'],\n",
    "               ['2701000','2702000','2703000','2704000','2707000','2705000','2706000','2708000'],\n",
    "               ['2801000','2802000','2803000','2804000','2805000','2806000','2807000','2808000','2810000','2811000'],\n",
    "               ['2901000','2902000','2903000','2904000','2905000'],\n",
    "               ['3001000','3002000','3003000','3004000','3005000'],\n",
    "               ['3101000','3102000','3103000','3104000'],\n",
    "               ['3602000','3603000'],\n",
    "               ['4137000','4104000','4105000','4107000','4108000','4109000','4110000','4112000','4113000','4114000','4115000','4116000','4133000','4118000','4120000','4126000','4122000','4123000','4132000','4124000','4127000','4128000','4101000','4106000','4129000','4111000','4130000','4117000','4119000','4121000','4154000','4125000','4150000','4152000'],\n",
    "               ['4106000','4129000','4111000','4130000','4117000','4119000','4121000','4154000','4125000','4150000','4152000'],\n",
    "               ['4201000','4202000','4203000','4204000','4205000','4206000','4207000','4208000','4213000','4209000','4214000','4211000','4210000','4215000','4216000','4217000'],\n",
    "               ['4301000','4302000','4303000','4304000','4305000','4306000','4307000','4308000','4309000','4310000','4311000','4312000'],\n",
    "               ['4402000','4416000','4403000','4404000','4405000','4406000','4407000','4409000','4412000','4410000','4413000','4401000','4408000','4417000','4421000','4420000'],\n",
    "               ['4501000','4502000','4503000','4504000','4505000','4506000','4507000','4508000','4509000','4510000'],\n",
    "               ['4601000','4602000','4603000','4604000','4605000','4606000','4607000','4608000','4609000','4610000','4611000','4612000','4613000','4615000'],\n",
    "               ['4715000','4716000','4703000','4704000','4705000','4706000','4707000','4708000','4709000','4710000','4711000','4714000','4718000','4719000','4717000','4713000','4701000','4712000'],\n",
    "               ['4804000','4806000','4807000','4809000','4810000','4811000','4821000','4814000','4813000','4819000','4817000','4818000','4822000','4820000','4812000','4815000','4826000','4827000'],\n",
    "               ['4823000','4824000','4825000'],\n",
    "               ['5001000','5002000','5003000','5004000']\n",
    "               ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "station_value=[\"1103000\",\"2601000\",\"2701000\",\"2801000\",\"2901000\",\"3001000\",\"3101000\",\n",
    "               \"3602000\",\"4137000\",\"4106000\",\"4201000\",\"4301000\",\"4402000\",\"4501000\",\n",
    "               \"4601000\",\"4715000\",\"4804000\",\"4823000\",\"5001000\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class nfds_crawling :\n",
    "    \n",
    "    def __init__(self, driver, url, from_,to_) :\n",
    "        self.master_area=[]\n",
    "        self.driver=driver\n",
    "        self.url = url\n",
    "        self.from_ = from_\n",
    "        self.to_ = to_\n",
    "        \n",
    "    def date_creation(self) :\n",
    "        start = datetime.datetime.strptime(self.from_, '%Y-%m-%d').date()\n",
    "        end = datetime.datetime.strptime(self.to_, '%Y-%m-%d').date()\n",
    "        day_range = (end-start).days + 1\n",
    "        dates=[]\n",
    "        for i in range(day_range) :\n",
    "            dates.append(start+datetime.timedelta(days=i))\n",
    "            dates[i] = dates[i].strftime('%Y-%m-%d')\n",
    "        self.dates = dates\n",
    "        \n",
    "    def submission(self) :\n",
    "        submit = self.driver.find_element_by_class_name(name='submit')\n",
    "        submit.submit()\n",
    "        \n",
    "    def multi_select(self, xpath, index) :\n",
    "        elem = self.driver.find_element_by_xpath(xpath)\n",
    "        select = Select(elem)                \n",
    "        options = select.options[index].text\n",
    "        select.select_by_index(index)        \n",
    "        return options\n",
    "    \n",
    "    def option_len(self, xpath) :\n",
    "        elem = self.driver.find_element_by_xpath(xpath)\n",
    "        select = Select(elem)\n",
    "        return len(select.options)\n",
    "    \n",
    "    def date_selection(self, date_obj) :\n",
    "        elem_start = self.driver.find_element_by_xpath('//*[@id=\"txtSdate\"]')\n",
    "        elem_end = self.driver.find_element_by_xpath('//*[@id=\"txtEdate\"]')\n",
    "        elem_start.clear()\n",
    "        elem_end.clear()\n",
    "        elem_start.send_keys(date_obj)\n",
    "        elem_end.send_keys(date_obj)\n",
    "        \n",
    "    def table_waiting(self, selector) :\n",
    "        try :\n",
    "            element_present = EC.presence_of_all_elements_located((By.CSS_SELECTOR, selector))\n",
    "            WebDriverWait(self.driver, 5).until(element_present)\n",
    "        except TimeoutException :\n",
    "            print('Internet Connection maybe lost at table')\n",
    "                \n",
    "    def option_waiting(self, xpath, text_) :\n",
    "        try :\n",
    "            element_present = EC.text_to_be_present_in_element_value((By.XPATH, xpath),text_)\n",
    "            WebDriverWait(self.driver, 5).until(element_present)\n",
    "        except TimeoutException :\n",
    "            print('Internet Connection maybe lost at option')\n",
    "\n",
    "    def option_staleness(self,elem) : \n",
    "        try :            \n",
    "            element_present = EC.staleness_of(element=elem)\n",
    "            WebDriverWait(self.driver, 5).until(element_present)\n",
    "        except TimeoutException :\n",
    "            print('Option has not property populated')\n",
    "            \n",
    "             \n",
    "    #Parsing Table Function\n",
    "    def parsing_table(self,date_obj,table_selector,*args) :\n",
    "        html = self.driver.page_source\n",
    "        soup = BeautifulSoup(html,'lxml')\n",
    "        Soup = soup.select(table_selector)        \n",
    "        for i in range(len(Soup)) :\n",
    "            temp = Soup[i].get_text(separator='|', strip=True)\n",
    "            #str인 temp를 list 작업화\n",
    "            temp = temp.split('|')\n",
    "            temp.extend(list(args))\n",
    "            temp.append(date_obj)\n",
    "            self.master_area.append(temp)\n",
    "            \n",
    "    # Parsing Table divided into 2 section\n",
    "    def parsing_two_table(self,date_obj,table_selector_1,table_selector_2,*args) :\n",
    "        html = self.driver.page_source\n",
    "        soup = BeautifulSoup(html,'lxml')\n",
    "        Soup_main = soup.select(table_selector_1)\n",
    "        Soup_body = soup.select(table_selector_2)\n",
    "    \n",
    "        for i in range(1,len(Soup_main)) :\n",
    "            temp_main = Soup_main[i].get_text(separator='|',strip=True)\n",
    "            temp_main = temp_main.split('|')\n",
    "            temp_body = Soup_body[i].get_text(separator='|',strip=True)\n",
    "            temp_body = temp_body.split('|')            \n",
    "            temp_main.extend(temp_body)\n",
    "            temp_main.extend(list(args))\n",
    "            temp_main.append(date_obj)\n",
    "            self.master_area.append(temp_main)\n",
    "        \n",
    "            \n",
    "    # 기본현황 - 시/도 (옵션 1개)\n",
    "    def area_table(self,area_xpath,table_selector) :\n",
    "        #GET URL\n",
    "        self.driver.get(self.url)\n",
    "        #SELECTING DATE\n",
    "        for i in range(len(self.dates)) :\n",
    "            self.date_selection(self.dates[i])\n",
    "            # SELECTING FIRST OPTION\n",
    "            for j in range(1, getattr(self,'option_len')(area_xpath)):\n",
    "                option = getattr(self,'multi_select')(area_xpath,j)\n",
    "                self.submission()\n",
    "                getattr(self,'table_waiting')(table_selector)                \n",
    "                getattr(self,'parsing_table')(self.dates[i],table_selector,option)\n",
    "                \n",
    "        return self.master_area\n",
    "    \n",
    "    # 기본현황 - 소방관서 (옵션 1개)\n",
    "    def fire_station(self, area_xpath_1,table_selector) :\n",
    "        #소방관서는 화재내역이 없을 경우 Table CSS Selector에 해당되는 테이블이\n",
    "        #아예 나타나지 않기에 해당 경우를 인터넷 연결이 끊긴 것과 별도로\n",
    "        #고려해야 함\n",
    "        #GET URL\n",
    "        self.driver.get(self.url)\n",
    "        # SELECTING DATE\n",
    "        for i in range(len(self.dates)):\n",
    "            self.date_selection(self.dates[i])\n",
    "            #SELECTING FIRST OPTION\n",
    "            for j in range(1,getattr(self,'option_len')(area_xpath_1)) :\n",
    "                option_a = getattr(self,'multi_select')(area_xpath_1,j)\n",
    "                self.submission()\n",
    "                getattr(self,'table_waiting')(table_selector)\n",
    "                getattr(self,'parsing_table')(self.dates[i],table_selector,option_a)                \n",
    "        \n",
    "        return self.master_area\n",
    "    \n",
    "    # 장소별-소방방화시설 (옵션 3개)\n",
    "    # 유일하게 두개의 table로 이루어졌기에 parsing_two_table 함수를 이용\n",
    "    def fire_facility(self,type_xpath,xpath_1,xpath_2,table_selector_1,table_selector_2) :\n",
    "        #GET URL\n",
    "        self.driver.get(self.url)\n",
    "        #SELECTING DATE\n",
    "        for i in range(len(self.dates)):\n",
    "            self.date_selection(self.dates[i])\n",
    "            #SELECTING TYPE(발생건수,비율,피해)\n",
    "            #2번째(j=1) 는 발생비율이므로 skip\n",
    "            for j in [0,2,3,4,5,6,7] :\n",
    "                option_a = getattr(self, 'multi_select')(type_xpath,j)\n",
    "                # 소방본부 선택\n",
    "                for k in range(1,getattr(self, 'option_len')(xpath_1)):\n",
    "                    elem = self.driver.find_element_by_xpath(xpath_2+'/option[1]')\n",
    "                    option_b=getattr(self,'multi_select')(xpath_1,k)\n",
    "                    getattr(self,'option_waiting')(xpath_2+'/option[2]',station_value[k-1])\n",
    "                    #소방서 선택\n",
    "                    for m in range(1,getattr(self,'option_len')(xpath_2)) :                                                \n",
    "                        option_c=getattr(self,'multi_select')(xpath_2,m)                        \n",
    "                        self.submission()\n",
    "                        getattr(self,'table_waiting')(table_selector_1)\n",
    "                        getattr(self,'table_waiting')(table_selector_2)\n",
    "                        getattr(self,'parsing_two_table')(self.dates[i],table_selector_1,table_selector_2,\n",
    "                                                          option_a,option_b,option_c)\n",
    "\n",
    "        return self.master_area\n",
    "    \n",
    "    # 장소별-다중이용업시설 (옵션 3개)\n",
    "    def fire_multi(self,type_xpath,xpath_1,xpath_2,table_selector) :\n",
    "        #GET URL\n",
    "        self.driver.get(self.url)\n",
    "        #SELECTING DATE\n",
    "        for i in range(len(self.dates)) :\n",
    "            self.date_selection(self.dates[i])\n",
    "        #SELECTING DATA TYPE(2번쨰는 발생비율로 SKIP)\n",
    "            for j in [0,2,3,4,5,6,7] :\n",
    "                option_a = getattr(self,'multi_select')(type_xpath,j)\n",
    "                for k in range(1,getattr(self,'option_len')(xpath_1)) :                    \n",
    "                    option_b = getattr(self,'multi_select')(xpath_1,k)                    \n",
    "                    getattr(self,'option_waiting')(xpath_2+'/option[2]',station_value[k-1])                    \n",
    "                    for m in range(1,getattr(self,'option_len')(xpath_2)) :\n",
    "                        option_c = getattr(self,'multi_select')(xpath_2,m)\n",
    "                        self.submission()\n",
    "                        getattr(self,'table_waiting')(table_selector)\n",
    "                        getattr(self,'parsing_table')(self.dates[i],table_selector,\n",
    "                                                     option_a,option_b,option_c)\n",
    "        return self.master_area\n",
    "    \n",
    "    # 장소별-건물구조별\n",
    "    def fire_bldg(self,type_xpath,xpath_1,xpath_2,table_selector) :\n",
    "        #GET URL\n",
    "        self.driver.get(self.url)\n",
    "        #SELECTING DATE\n",
    "        for i in range(len(self.dates)) :\n",
    "            self.date_selection(self.dates[i])\n",
    "        #SELECTING DATA TYPE(2번쨰는 발생비율로 SKIP)\n",
    "            for j in [0,2,3,4,5,6,7] :\n",
    "                option_a = getattr(self,'multi_select')(type_xpath,j)\n",
    "                for k in range(1,getattr(self,'option_len')(xpath_1)) :\n",
    "                    option_b = getattr(self,'multi_select')(xpath_1,k)\n",
    "                    getattr(self,'option_waiting')(xpath_2+'/option[2]',station_value[k-1])\n",
    "                    for m in range(1,getattr(self,'option_len')(xpath_2)) :\n",
    "                        option_c = getattr(self,'multi_select')(xpath_2,m)\n",
    "                        self.submission()\n",
    "                        getattr(self,'table_waiting')(table_selector)\n",
    "                        getattr(self,'parsing_table')(self.dates[i],table_selector,\n",
    "                                                     option_a,option_b,option_c)\n",
    "        return self.master_area\n",
    "    \n",
    "    #발화요인별-발화요인/장소별 (옵션 3개)\n",
    "    def fire_rsn_lc(self,type_xpath,xpath_1,xpath_2,table_selector) :\n",
    "        #GET URL\n",
    "        self.driver.get(self.url)\n",
    "        #SELECTING DATE\n",
    "        for i in range(len(self.dates)) :\n",
    "            self.date_selection(self.dates[i])\n",
    "        #SELECTING DATA TYPE(2번쨰는 발생비율로 SKIP)\n",
    "            for j in [0,2,3,4,5,6,7] :\n",
    "                option_a = getattr(self,'multi_select')(type_xpath,j)\n",
    "                for k in range(1,getattr(self,'option_len')(xpath_1)) :\n",
    "                    option_b = getattr(self,'multi_select')(xpath_1,k)\n",
    "                    getattr(self,'option_waiting')(xpath_2+'/option[2]',station_value[k-1])\n",
    "                    for m in range(1,getattr(self,'option_len')(xpath_2)) :\n",
    "                        option_c = getattr(self,'multi_select')(xpath_2,m)\n",
    "                        self.submission()\n",
    "                        getattr(self,'table_waiting')(table_selector)\n",
    "                        getattr(self,'parsing_table')(self.dates[i],table_selector,\n",
    "                                                     option_a,option_b,option_c)\n",
    "        return self.master_area\n",
    "    \n",
    "    #발화요인별-연소확대사유현황 (옵션 3개)\n",
    "    def fire_aggv_rsn(self,type_xpath,xpath_1,xpath_2,table_selector) :\n",
    "        #GET URL\n",
    "        self.driver.get(self.url)\n",
    "        #SELECTING DATE\n",
    "        for i in range(len(self.dates)) :\n",
    "            self.date_selection(self.dates[i])\n",
    "        #SELECTING DATA TYPE(2번쨰는 발생비율로 SKIP)\n",
    "            for j in [0,2,3,4,5,6,7] :\n",
    "                option_a = getattr(self,'multi_select')(type_xpath,j)\n",
    "                for k in range(1,getattr(self,'option_len')(xpath_1)) :\n",
    "                    option_b = getattr(self,'multi_select')(xpath_1,k)\n",
    "                    getattr(self,'option_waiting')(xpath_2+'/option[2]',station_value[k-1])\n",
    "                    for m in range(1,getattr(self,'option_len')(xpath_2)) :\n",
    "                        option_c = getattr(self,'multi_select')(xpath_2,m)\n",
    "                        self.submission()\n",
    "                        getattr(self,'table_waiting')(table_selector)\n",
    "                        getattr(self,'parsing_table')(self.dates[i],table_selector,\n",
    "                                                     option_a,option_b,option_c)\n",
    "        return self.master_area"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#연소확대사유 테스트\n",
    "firefox.get(url_aggv_rsn)\n",
    "test=nfds_crawling(firefox,url_aggv_rsn,'2017-09-01','2017-09-30')\n",
    "test.date_creation()\n",
    "test_body=test.fire_aggv_rsn(type_xpath_aggv_rsn,xpath_aggv_rsn_1,xpath_aggv_rsn_2,table_selector_aggv_rsn)\n",
    "test_df = pd.DataFrame(test_body,columns=columns_aggv_rsn)\n",
    "test_df.to_csv(index=False,encoding='utf-8-sig',path_or_buf=r'C:\\Users\\Administrator\\Desktop\\새 폴더\\fire_aggv_rsn_20170901_20170930.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#발화요인 장소별 테스트\n",
    "chrome.get(url_rsn_lc)\n",
    "test=nfds_crawling(chrome,url_rsn_lc,'2017-09-01','2017-09-30')\n",
    "test.date_creation()\n",
    "test_body=test.fire_rsn_lc(type_xpath_rsn_lc,xpath_rsn_lc_1,xpath_rsn_lc_2,table_selector_rsn_lc)\n",
    "test_df = pd.DataFrame(test_body,columns=columns_rsn_lc)\n",
    "test_df.to_csv(index=False,encoding='utf-8-sig',path_or_buf=r'C:\\Users\\Administrator\\Desktop\\새 폴더\\fire_rsn_lc_20170901_20170930.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#소방방화시설 테스트\n",
    "firefox.get(url_facility)\n",
    "test=nfds_crawling(firefox,url_facility,'2017-09-01','2017-09-30')\n",
    "test.date_creation()\n",
    "test_body=test.fire_facility(type_xpath_facility,xpath_facility_1,xpath_facility_2,table_selector_facility_1,table_selector_facility_2)\n",
    "test_df = pd.DataFrame(test_body,columns=columns_facility)\n",
    "test_df.to_csv(index=False,encoding='utf-8-sig',path_or_buf=r'C:\\Users\\Administrator\\Desktop\\새 폴더\\fire_facility_20170901_20170930.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#다중이용시설 테스트\n",
    "test=nfds_crawling(chrome,url_multi,'2017-09-01','2017-09-30')\n",
    "test.date_creation()\n",
    "test_body=test.fire_multi(type_xpath_multi,xpath_multi_1,xpath_multi_2,table_selector_multi)\n",
    "test_df = pd.DataFrame(test_body,columns=columns_multi)\n",
    "test_df.to_csv(index=False,encoding='utf-8-sig',path_or_buf=r'C:\\Users\\Administrator\\Desktop\\새 폴더\\fire_multi_20170901_20170930.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#건물유형 테스트\n",
    "test=nfds_crawling(chrome,url_bldg,'2017-09-01','2017-09-30')\n",
    "test.date_creation()\n",
    "test_body=test.fire_bldg(type_xpath_bldg,xpath_bldg_1,xpath_bldg_2,table_selector_bldg)\n",
    "test_df = pd.DataFrame(test_body,columns=columns_bldg)\n",
    "test_df.to_csv(index=False,encoding='utf-8-sig',path_or_buf=r'C:\\Users\\Administrator\\Desktop\\새 폴더\\fire_bldg_20170901_20170930.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#시도 Test\n",
    "js.get(url_area)\n",
    "test=nfds_crawling(js,url_area,'2017-09-01','2017-09-30')\n",
    "test.date_creation()\n",
    "test_body=test.area_table(xpath_area,table_selector_area)\n",
    "test_df = pd.DataFrame(test_body,columns=columns_area)\n",
    "test_df.to_csv(index=False,encoding='utf-8-sig',path_or_buf=r'C:\\Users\\Administrator\\Desktop\\새 폴더\\fire_area_20170901_20170930.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#소방관서 Test\n",
    "chrome.get(url_station)\n",
    "test=nfds_crawling(chrome,url_station,'2017-09-01','2017-09-30')\n",
    "test.date_creation()\n",
    "test_body=test.area_table(xpath_station_1,table_selector_station)\n",
    "test_df = pd.DataFrame(test_body,columns=columns_area)\n",
    "test_df.to_csv(index=False,encoding='utf-8-sig',path_or_buf=r'C:\\Users\\Administrator\\Desktop\\새 폴더\\fire_station_20170901_20170930.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}