selenium-crawler.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["from selenium import webdriver\n","from selenium.webdriver.support.ui import Select\n","from selenium.webdriver.common.by import By\n","from selenium.webdriver.chrome.service import Service as ChromeService\n","from selenium.webdriver.chrome.options import Options\n","from selenium.webdriver.support import expected_conditions as EC\n","from webdriver_manager.chrome import ChromeDriverManager\n","from PIL import Image\n","from io import BytesIO\n","from time import sleep\n","from pathlib import Path\n","import pytesseract, csv"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# variables used in the crawler\n","allLabLinks = []\n","url = \"https://ma.mohw.gov.tw/masearch/\"\n","targetType = '牙體技術所'\n","linkTxt = '詳細資料'\n","\n","# css ID selector\n","captchaImgID = 'ctl00_ContentPlaceHolder1_ImageCheck'\n","typeID = 'ctl00_ContentPlaceHolder1_ddlBAS_KIND'\n","captchaBoxID = 'ctl00_ContentPlaceHolder1_TextBox1'\n","submitID = 'ctl00_ContentPlaceHolder1_btnSearch'\n","pageID = 'ctl00_ContentPlaceHolder1_NetPager1_lblCurrentIndex'\n","totalID = 'ctl00_ContentPlaceHolder1_NetPager1_lblPageCount'\n","nextButtonID = 'ctl00_ContentPlaceHolder1_NetPager1_lnkbtnNext'"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["options = Options()\n","options.add_argument('--headless')\n","options.add_argument('--disable-gpu')\n","\n","driver = webdriver.Chrome(\n","    service=ChromeService(ChromeDriverManager().install()),\n","    options=options\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["driver.get(url)\n","\n","orgType = Select(driver.find_element(By.ID, typeID))\n","orgType.select_by_visible_text(targetType)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def tryCaptcha():\n","    '''try resolve the captcha, re-try every 2 sec if not success'''\n","    captchaPngData = driver.find_element(By.ID, captchaImgID).screenshot_as_png\n","    imgObj = Image.open(BytesIO(captchaPngData))\n","    captchaTxt = pytesseract.image_to_string(imgObj).strip()\n","    driver.find_element(By.ID, captchaBoxID).send_keys(captchaTxt)\n","    driver.find_element(By.ID, submitID).click()\n","    try:\n","        driver.switch_to.alert.accept()\n","        sleep(2)\n","        tryCaptcha()\n","    except:\n","        return"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["tryCaptcha()"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["totalPage = int(\n","    driver.find_element(By.ID, totalID).text.removeprefix(\"共 \").removesuffix(\" 頁\")\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["while True:\n","    try:\n","        page = int(\n","            driver.find_element(By.ID, pageID).text.removeprefix(\"第 \").removesuffix(\" 頁\")\n","        )\n","        links = driver.find_elements(By.LINK_TEXT, linkTxt)\n","        hrefs = [l.get_attribute(\"href\") for l in links]\n","    except:\n","        # may fail due to network problem\n","        break\n","    allLabLinks += hrefs\n","    print('processed:')\n","    print(page, end=', ')\n","    if page >= totalPage:\n","        print('done!')\n","        break\n","    driver.find_element(By.ID, nextButtonID).click()\n","    sleep(1)\n","\n","# driver.close()"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# export all lab links to a text file\n","Path(\"labs.txt\").write_text('\\n'.join(allLabLinks))"]},{"cell_type":"markdown","metadata":{},"source":[" ## crawl lab details"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# CSS selectors to search for\n","selectors = [\n","    \"ctl00_ContentPlaceHolder1_lblBAS_ID\",\n","    \"ctl00_ContentPlaceHolder1_lblBAS_NAME\",\n","    \"ctl00_ContentPlaceHolder1_lblBAS_AREA\",\n","    \"ctl00_ContentPlaceHolder1_lblBAS_STATUS\",\n","    \"ctl00_ContentPlaceHolder1_lblBAS_ADDRESS\",\n","    \"ctl00_ContentPlaceHolder1_lblBAS_TEL\",\n","]"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["with open('lab_details.csv', 'w') as f:\n","    w = csv.writer(f, dialect='unix')\n","    w.writerow('id name area status address tel'.split())\n","    for link in allLabLinks:\n","        try:\n","            driver.get(link)\n","            row = [driver.find_element(By.ID, s).text for s in selectors]\n","            print(row[0], )   # id\n","            w.writerow(row)\n","        except:\n","            # maybe network problem, whatever error\n","            break\n","\n","driver.close()"]}],"metadata":{"kernelspec":{"display_name":"Python 3.11.0 ('.venv': poetry)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.0"},"orig_nbformat":4,"vscode":{"interpreter":{"hash":"c3530e6592aef8d1b52674b97beb76f66248d0e1646718b45855c471315e3ba5"}}},"nbformat":4,"nbformat_minor":2}