爬蟲程式.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 爬蟲程式"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Google_API\n",
    "這部分是採用google提供的place API資訊，參酌網址如下：https://developers.google.com/places/web-service/overview\n",
    "透過API提供的資料，我們收集了不同地區內的各個餐廳店家名稱、地址、電話、評分、價位、連結、評價數、經緯度等資訊。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 139
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 30133,
     "status": "ok",
     "timestamp": 1583825378030,
     "user": {
      "displayName": "曾文昱",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiaYleYj67NwrHnWON5O-ST9Mbtkw5_c1N8YrfT=s64",
      "userId": "10065332786568642768"
     },
     "user_tz": -480
    },
    "id": "EZrYu04IdaFk",
    "outputId": "aa456f61-a0a1-4bec-9306-fd75b1eb04ed"
   },
   "outputs": [],
   "source": [
    "import requests, json, csv\n",
    "from urllib.parse import urlencode\n",
    "api_key = \"這裡要放入從google platform申請的API金鑰\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1 店家清單搜尋\n",
    "這部分利用了place API的nearby search功能。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Places API\n",
    "def searching_restaurants(content, key, rad_input = 100):\n",
    "    \"\"\"根據輸入的內容與範圍搜尋店家\"\"\"\n",
    "    base_endpoint_places = \"https://maps.googleapis.com/maps/api/place/nearbysearch/json\"\n",
    "    params = {\n",
    "        \"key\" : key,\n",
    "        \"location\" : \"25.012606331913954, 121.53534692174303\",  #以特定經緯度地點搜尋\n",
    "        \"radius\" : rad_input,  #方圓半徑\n",
    "        \"keyword\" : content  #搜尋內容\n",
    "    }\n",
    "    params_encoded = urlencode(params)\n",
    "    places_endpoint = f\"{base_endpoint_places}?{params_encoded}\"\n",
    "\n",
    "    req = requests.get(places_endpoint)\n",
    "    dta = req.json()\n",
    "    re_list = dta[\"results\"]\n",
    "    place_id_list = []\n",
    "    for i in re_list:\n",
    "        place_id_list.append(i['place_id'])\n",
    "    return place_id_list\n",
    "\n",
    "#關鍵字搜尋：根據各種食物的類型搜尋店家，並傳回各種類型的店家清單\n",
    "place_dictionary = dict()\n",
    "Keyword_list = [\"義大利麵\", \"咖哩\", \"牛排\", \"粥\", \"冰品\", \"飲料\", \"火鍋\", \"水餃\", \"小吃\", \"炒飯\", \"日式料理\", \"韓式料理\", \"泰式料理\", \"滷味\", \"鹹酥雞\", \"便當\", \"早午餐\", \"低GI\", \"拉麵\", \"素食\", \"速食\"]\n",
    "for i in Keyword_list:\n",
    "    dta_list = searching_restaurants(i, api_key, rad_input = 500)\n",
    "    place_dictionary[i] = dta_list\n",
    "print(place_dictionary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.2 店家詳細資料搜尋\n",
    "這部分則是根據1.1 傳回的清單，用place API中的details功能，找尋店家的詳細資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 各店家細節csv\n",
    "def search_for_details(place, goo_key):\n",
    "    '''利用place API找尋給定店家的細節資訊'''\n",
    "    place_id = place\n",
    "    detail_base_endpoint = \"https://maps.googleapis.com/maps/api/place/details/json\"\n",
    "    detail_params = {\n",
    "        \"place_id\" : f\"{place_id}\",\n",
    "        \"language\" : \"zh-TW\",\n",
    "        \"fields\" : \"name,formatted_address,geometry,rating,formatted_phone_number,url,opening_hours,rating,review,user_ratings_total\",  #要收集的資訊\n",
    "        \"key\" : goo_key\n",
    "    }\n",
    "    detail_params_encoded = urlencode(detail_params)\n",
    "    detail_url = f\"{detail_base_endpoint}?{detail_params_encoded}\"\n",
    "    req_detail = requests.get(detail_url)\n",
    "    req_dict = req_detail.json()\n",
    "    detail_dict = req_dict['result']\n",
    "    \n",
    "    #整理要傳回的資訊\n",
    "    for_return = []\n",
    "    for i in ['name', 'formatted_address', 'formatted_phone_number', 'rating', 'price_level', 'url', 'user_ratings_total']:\n",
    "        try:\n",
    "            for_return.append(detail_dict[i])\n",
    "        except KeyError:\n",
    "            for_return.append('null')\n",
    "    for_return.append(detail_dict['geometry']['location']['lat'])\n",
    "    for_return.append(detail_dict['geometry']['location']['lng'])\n",
    "    return for_return\n",
    "\n",
    "#讀取places資料\n",
    "Keyword_list = [\"義大利麵\", \"咖哩\", \"牛排\", \"粥\", \"冰品\", \"飲料\", \"火鍋\", \"水餃\", \"小吃\", \"炒飯\", \"日式料理\", \"韓式料理\", \"泰式料理\", \"滷味\", \"鹹酥雞\", \"便當\", \"早午餐\", \"低GI\", \"拉麵\", \"素食\", \"速食\"]\n",
    "type_dict = {\"義大利麵\":'西式/美式', \"咖哩\":'日式料理', \"牛排\":'西式/美式', \"粥\":\"台式/小吃\", \"冰品\":'飲料/甜點', \"飲料\":'飲料/甜點', \"火鍋\":'火鍋', \"水餃\":'台式/小吃', \"小吃\":'台式/小吃', \"炒飯\":'台式/小吃', \"日式料理\":\"日式料理\", \"韓式料理\":\"韓式料理\", \"泰式料理\":\"東南亞\", \"滷味\":\"台式/小吃\", \"鹹酥雞\":'台式/小吃', \"便當\":\"台式/小吃\", \"早午餐\":'西式/美式', \"低GI\":'素食/健康', \"拉麵\":'日式料理', \"素食\":\"素食/健康\", \"速食\":'速食'}\n",
    "fh1 = open('gongguan_place_id.txt', 'r', encoding = 'utf-8')\n",
    "fhstr = fh1.read()\n",
    "fhdic = eval(fhstr)\n",
    "fh1.close()\n",
    "\n",
    "#讀取details，存成二維list\n",
    "total_list = []\n",
    "for i in Keyword_list:\n",
    "    places_list = fhdic[i]\n",
    "    for aplace in places_list:\n",
    "        list_for_a_place = [type_dict[i], i, aplace]\n",
    "        place_detail = search_for_details(aplace, api_key)\n",
    "        list_for_a_place.extend(place_detail)\n",
    "        total_list.append(list_for_a_place)\n",
    "\n",
    "#寫入csv檔案\n",
    "with open('gongguan_new_details.csv', 'w', newline='', errors = 'ignore') as csvfile:\n",
    "    writer = csv.writer(csvfile)\n",
    "    writer.writerow(['general_type', 'specific_type', 'place_id', '店名', '地址', '電話', '評分', '價位', '連結', '評價數', '緯度', '經度'])\n",
    "    for i in total_list:\n",
    "        writer.writerow(i)\n",
    "    csvfile.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.3 店家開放時間、繁忙時段 \n",
    "在google地圖當中，能夠看到各個店家的開放時間與每個小時的繁忙時段。然而，google目前並未直接透過API提供各店家的繁忙時段資訊。因此，這邊使用了另一個populartimes的套件，抓取不同店家的繁忙時段資料，而營業時段的資料則仍維持使用place API。\n",
    "source: https://github.com/m-wrzr/populartimes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install git+https://github.com/m-wrzr/populartimes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import populartimes\n",
    "#各店家開放時間、populartime\n",
    "def search_for_times(place, goo_key):\n",
    "    '''利用place API找尋給定店家的開放時間與繁忙時段'''\n",
    "    #places API\n",
    "    place_id = place\n",
    "    detail_base_endpoint = \"https://maps.googleapis.com/maps/api/place/details/json\"\n",
    "    detail_params = {\n",
    "        \"place_id\" : f\"{place_id}\",\n",
    "        \"language\" : \"zh-TW\",\n",
    "        \"fields\" : \"name,opening_hours\",  #要收集的資訊\n",
    "        \"key\" : goo_key\n",
    "    }\n",
    "    detail_params_encoded = urlencode(detail_params)\n",
    "    detail_url = f\"{detail_base_endpoint}?{detail_params_encoded}\"\n",
    "    req_detail = requests.get(detail_url)\n",
    "    req_dict = req_detail.json()\n",
    "    detail_dict = req_dict['result']\n",
    "    \n",
    "    #populartimes\n",
    "    pop_det = populartimes.get_id(goo_key, place)\n",
    "    try:\n",
    "        pop_list = pop_det['populartimes']\n",
    "    except KeyError:\n",
    "        pop_list = [{'data' : 'null'}, {'data' : 'null'}, {'data' : 'null'}, {'data' : 'null'}, {'data' : 'null'}, {'data' : 'null'}, {'data' : 'null'}]\n",
    "\n",
    "    #整理要傳回的資訊\n",
    "    for_return = []\n",
    "    for_return.append(detail_dict['name'])\n",
    "    try:\n",
    "        for_return.append(detail_dict['opening_hours']['periods'])\n",
    "    except KeyError:\n",
    "            for_return.append('null')\n",
    "    try:\n",
    "        for_return.append(detail_dict['opening_hours']['weekday_text'])\n",
    "    except KeyError:\n",
    "            for_return.append('null')\n",
    "    for i in range(0, 7):\n",
    "        for_return.append(pop_list[i]['data'])\n",
    "    return for_return\n",
    "\n",
    "#讀取places資料\n",
    "Keyword_list = [\"義大利麵\", \"咖哩\", \"牛排\", \"粥\", \"冰品\", \"飲料\", \"火鍋\", \"水餃\", \"小吃\", \"炒飯\", \"日式料理\", \"韓式料理\", \"泰式料理\", \"滷味\", \"鹹酥雞\", \"便當\", \"早午餐\", \"低GI\", \"拉麵\", \"素食\", \"速食\"]\n",
    "type_dict = {\"義大利麵\":'西式/美式', \"咖哩\":'日式料理', \"牛排\":'西式/美式', \"粥\":\"台式/小吃\", \"冰品\":'飲料/甜點', \"飲料\":'飲料/甜點', \"火鍋\":'火鍋', \"水餃\":'台式/小吃', \"小吃\":'台式/小吃', \"炒飯\":'台式/小吃', \"日式料理\":\"日式料理\", \"韓式料理\":\"韓式料理\", \"泰式料理\":\"東南亞\", \"滷味\":\"台式/小吃\", \"鹹酥雞\":'台式/小吃', \"便當\":\"台式/小吃\", \"早午餐\":'西式/美式', \"低GI\":'素食/健康', \"拉麵\":'日式料理', \"素食\":\"素食/健康\", \"速食\":'速食'}\n",
    "fh1 = open('gongguan_new_id.txt', 'r', encoding = 'utf-8')\n",
    "fhstr = fh1.read()\n",
    "fhdic = eval(fhstr)\n",
    "fh1.close()\n",
    "#讀取details，存成二維list\n",
    "total_list = []\n",
    "for i in Keyword_list:\n",
    "    places_list = fhdic[i]\n",
    "    for aplace in places_list:\n",
    "        list_for_a_place = [type_dict[i], i, aplace]\n",
    "        place_detail = search_for_times(aplace, api_key)\n",
    "        list_for_a_place.extend(place_detail)\n",
    "        total_list.append(list_for_a_place)\n",
    "\n",
    "#寫入csv檔案\n",
    "with open('gongguan_time_details.csv', 'w', newline='', errors = 'ignore') as csvfile:\n",
    "    writer = csv.writer(csvfile)\n",
    "    writer.writerow(['general_type', 'specific_type', 'place_id', '店名', 'periods', 'weekday_text', 'Mon', 'Tues', 'Wed', 'Thur', 'Fri', 'Sat', 'Sun'])\n",
    "    for i in total_list:\n",
    "        writer.writerow(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.4 google評論"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reviews_from_google\n",
    "def search_for_google_reviews(place, goo_key):\n",
    "    '''利用place API找尋給定店家的開放時間與繁忙時段'''\n",
    "    #places API\n",
    "    place_id = place\n",
    "    detail_base_endpoint = \"https://maps.googleapis.com/maps/api/place/details/json\"\n",
    "    detail_params = {\n",
    "        \"place_id\" : f\"{place_id}\",\n",
    "        \"language\" : \"zh-TW\",\n",
    "        \"fields\" : \"name,review\",  #要收集的資訊\n",
    "        \"key\" : goo_key\n",
    "    }\n",
    "    detail_params_encoded = urlencode(detail_params)\n",
    "    detail_url = f\"{detail_base_endpoint}?{detail_params_encoded}\"\n",
    "    req_detail = requests.get(detail_url)\n",
    "    req_dict = req_detail.json()\n",
    "    detail_dict = req_dict['result']\n",
    "    \n",
    "    #整理要傳回的資訊\n",
    "    for_return = []\n",
    "    for_return.append(detail_dict['name'])\n",
    "    try:\n",
    "        for are in detail_dict['reviews']:   \n",
    "            for_return.append(are['text'])\n",
    "    except KeyError:\n",
    "            for_return.append('null')\n",
    "    return for_return\n",
    "\n",
    "#讀取places資料\n",
    "Keyword_list = [\"義大利麵\", \"咖哩\", \"牛排\", \"粥\", \"冰品\", \"飲料\", \"火鍋\", \"水餃\", \"小吃\", \"炒飯\", \"日式料理\", \"韓式料理\", \"泰式料理\", \"滷味\", \"鹹酥雞\", \"便當\", \"早午餐\", \"低GI\", \"拉麵\", \"素食\", \"速食\"]\n",
    "type_dict = {\"義大利麵\":'西式/美式', \"咖哩\":'日式料理', \"牛排\":'西式/美式', \"粥\":\"台式/小吃\", \"冰品\":'飲料/甜點', \"飲料\":'飲料/甜點', \"火鍋\":'火鍋', \"水餃\":'台式/小吃', \"小吃\":'台式/小吃', \"炒飯\":'台式/小吃', \"日式料理\":\"日式料理\", \"韓式料理\":\"韓式料理\", \"泰式料理\":\"東南亞\", \"滷味\":\"台式/小吃\", \"鹹酥雞\":'台式/小吃', \"便當\":\"台式/小吃\", \"早午餐\":'西式/美式', \"低GI\":'素食/健康', \"拉麵\":'日式料理', \"素食\":\"素食/健康\", \"速食\":'速食'}\n",
    "fh1 = open('gongguan_new_id.txt', 'r', encoding = 'utf-8')\n",
    "fhstr = fh1.read()\n",
    "fhdic = eval(fhstr)\n",
    "fh1.close()\n",
    "\n",
    "#讀取reviews，存成二維list\n",
    "total_list = []\n",
    "for i in Keyword_list:\n",
    "    places_list = fhdic[i]\n",
    "    for aplace in places_list:\n",
    "        list_for_a_place = [type_dict[i], i, aplace]\n",
    "        reviews = search_for_google_reviews(aplace, api_key)\n",
    "        list_for_a_place.extend(reviews)\n",
    "        total_list.append(list_for_a_place)\n",
    "\n",
    "#寫入csv檔案\n",
    "with open('gongguan_google_reviews.csv', 'w', newline='', errors = 'ignore') as csvfile:\n",
    "    writer = csv.writer(csvfile)\n",
    "    for i in total_list:\n",
    "        writer.writerow(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Instagram的評論、圖片搜尋\n",
    "這部分主要是使用selenium幫助我們自動化的抓取資料(各個地點的ig評論、照片、連結)，並進行一些基礎的資料清理後，儲存成csv檔案"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import time\n",
    "import csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1 IG的位置連結 \n",
    "根據商家的名稱，搜尋INSTAGRAM的各個地點連結！"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#根據店家名稱抓取location路徑\n",
    "URI='https://www.instagram.com'\n",
    "driver=webdriver.Chrome()\n",
    "driver.get(URI)\n",
    "\n",
    "#登入\n",
    "time.sleep(1)\n",
    "accounts = driver.find_element_by_xpath('//*[@name=\"username\"]')\n",
    "time.sleep(1)\n",
    "accounts.send_keys(\"\")  #你的帳號名稱\n",
    "time.sleep(1)\n",
    "pw = driver.find_element_by_xpath('//*[@name=\"password\"]')\n",
    "time.sleep(1)\n",
    "pw.send_keys(\"\")  #你的密碼\n",
    "login_btn = driver.find_element_by_xpath(\"//button[@class='sqdOP  L3NKy   y3zKF     ']\")\n",
    "login_btn.click()\n",
    "time.sleep(3)\n",
    "\n",
    "#是否自動登入：選擇稍後再說\n",
    "later_btn = driver.find_element_by_xpath(\"//button[@class='sqdOP yWX7d    y3zKF     ']\")\n",
    "later_btn.click()\n",
    "time.sleep(2)\n",
    "\n",
    "#是否開啟通知：選擇稍後再說\n",
    "later_btn_2 = driver.find_element_by_xpath(\"//button[@class='aOOlW   HoLwm ']\")\n",
    "later_btn_2.click()\n",
    "time.sleep(2)\n",
    "\n",
    "def search_for_href(place_name):\n",
    "    driver.get(URI)\n",
    "    time.sleep(1)\n",
    "    \n",
    "    #搜尋資料\n",
    "    search = driver.find_element_by_xpath(\"//input[@placeholder='搜尋']\")\n",
    "    search.clear() \n",
    "    search.send_keys(place_name)\n",
    "    search.send_keys(Keys.ENTER) \n",
    "    time.sleep(1)\n",
    "\n",
    "    #擷取連結\n",
    "    try:\n",
    "        first_re = driver.find_element_by_xpath(\"//div[@class='nebtz coreSpriteLocation']/ancestor::a\")\n",
    "    except:\n",
    "        try:\n",
    "            time.sleep(5)\n",
    "            first_re = driver.find_element_by_xpath(\"//div[@class='nebtz coreSpriteLocation']/ancestor::a\")\n",
    "        except:\n",
    "            return \"null\"\n",
    "    try: \n",
    "        link = first_re.get_attribute(\"href\")\n",
    "    except:\n",
    "        link = 'null'\n",
    "    return link\n",
    "\n",
    "#執行搜尋\n",
    "list_of_href = []\n",
    "with open('./crawling/gongguan/gongguan_details.csv', newline='') as file_cont:  #讀取商家名稱資料\n",
    "    # 讀取 CSV 檔案內容\n",
    "    content = csv.reader(file_cont)\n",
    "    content = list(content)\n",
    "    for row in range(1, len(content)):\n",
    "        single_id = content[row][2]\n",
    "        single_name = content[row][3]\n",
    "        list_of_href.append([])\n",
    "        list_of_href[-1].append(single_id)\n",
    "        list_of_href[-1].append(single_name)\n",
    "        list_of_href[-1].append(search_for_href(single_name))\n",
    "        print(list_of_href[-1])\n",
    "#輸出資料\n",
    "print(list_of_href)\n",
    "file_cont.close()\n",
    "\n",
    "#寫入csv檔案\n",
    "with open('gongguan_IG_links.csv', 'w', newline='', errors = 'ignore') as csvfile:\n",
    "    writer = csv.writer(csvfile)\n",
    "    writer.writerow(['place_id', 'name', 'IG_link'])\n",
    "    for i in list_of_href:\n",
    "        writer.writerow(i)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.2 IG圖片與連結 \n",
    "找尋前述連結內的熱門貼文，並從中抓取圖片、附上圖片來源。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#搜尋各location內的圖片\n",
    "URI='https://www.instagram.com'\n",
    "driver=webdriver.Chrome()\n",
    "driver.get(URI)\n",
    "\n",
    "#登入\n",
    "time.sleep(1)\n",
    "accounts = driver.find_element_by_xpath('//*[@name=\"username\"]')\n",
    "time.sleep(1)\n",
    "accounts.send_keys(\"fish._.03xx\")\n",
    "time.sleep(1)\n",
    "pw = driver.find_element_by_xpath('//*[@name=\"password\"]')\n",
    "time.sleep(1)\n",
    "pw.send_keys(\"zxcvbnm2536\")\n",
    "login_btn = driver.find_element_by_xpath(\"//button[@class='sqdOP  L3NKy   y3zKF     ']\")\n",
    "login_btn.click()\n",
    "time.sleep(3)\n",
    "\n",
    "#是否自動登入：選擇稍後再說\n",
    "later_btn = driver.find_element_by_xpath(\"//button[@class='sqdOP yWX7d    y3zKF     ']\")\n",
    "later_btn.click()\n",
    "time.sleep(2)\n",
    "\n",
    "#是否開啟通知：選擇稍後再說\n",
    "later_btn_2 = driver.find_element_by_xpath(\"//button[@class='aOOlW   HoLwm ']\")\n",
    "later_btn_2.click()\n",
    "time.sleep(2)\n",
    "\n",
    "\n",
    "def search_for_post_link(location):\n",
    "    '''從location中抓出貼文連結'''\n",
    "    driver.get(location)\n",
    "    time.sleep(1)\n",
    "\n",
    "    #擷取連結\n",
    "    try:\n",
    "        link_ob = driver.find_element_by_xpath(\"//div[@class='v1Nh3 kIKUG  _bz0w']/a\")\n",
    "    except:\n",
    "        try:\n",
    "            time.sleep(5)\n",
    "            link_ob = driver.find_element_by_xpath(\"//div[@class='v1Nh3 kIKUG  _bz0w']/a\")\n",
    "        except:\n",
    "            return \"None\"\n",
    "    try: \n",
    "        href = link_ob.get_attribute(\"href\")\n",
    "    except:\n",
    "        href = 'None'\n",
    "    return href\n",
    "\n",
    "def search_for_img(place_id, post_link):\n",
    "    driver.get(a_link)\n",
    "    time.sleep(3)\n",
    "    try: \n",
    "        time.sleep(1)\n",
    "        img_tag = driver.find_element_by_xpath(\"//div[@class='eLAPa RzuR0']/div/img\")\n",
    "        img_href = img_tag.get_attribute(\"src\")\n",
    "        resp = requests.get(img_href, stream=True)\n",
    "        local_file = open(str(place_id) + '.jpg', 'wb')\n",
    "        shutil.copyfileobj(resp.raw, local_file)\n",
    "        print(\"completed!\")\n",
    "        del resp\n",
    "        time.sleep(3)\n",
    "        local_file.close()\n",
    "        return True\n",
    "    except:\n",
    "        return False\n",
    "\n",
    "#開啟IG location檔案\n",
    "with open('118_link.csv', newline='') as csvFile:\n",
    "    rows = csv.reader(csvFile)\n",
    "    rows = list(rows)\n",
    "    location_list = []\n",
    "    for i in range(1, len(rows)):\n",
    "        place_id = rows[i][0]\n",
    "        name = rows[i][1]\n",
    "        ig_link = rows[i][2]\n",
    "        location_list.append([place_id, name, ig_link])\n",
    "csvFile.close()\n",
    "\n",
    "new_list = []\n",
    "for a_loc in location_list:\n",
    "    if a_loc[2] == 'null':\n",
    "        new_list.append(a_loc)\n",
    "        new_list[-1].append(\"None\")\n",
    "        continue\n",
    "    else:\n",
    "        #確定該地點的確有文章\n",
    "        try: \n",
    "            a_link = search_for_post_link(a_loc[2])\n",
    "        except:\n",
    "            new_list.append(a_loc)\n",
    "            new_list[-1].append(\"None\")\n",
    "            continue\n",
    "        \n",
    "        #載入圖片\n",
    "        if search_for_img(a_loc[0], a_link) == True:\n",
    "            new_list.append(a_loc)\n",
    "            new_list[-1].append(a_link)\n",
    "        else:\n",
    "            new_list.append(a_loc)\n",
    "            new_list[-1].append(\"None\")\n",
    "with open('118_img.csv','w', newline='') as csvFile:\n",
    "    writer = csv.writer(csvFile)\n",
    "    writer.writerow(['place_id', 'name', 'IG_link', 'img_source'])\n",
    "    for i in new_list:\n",
    "        writer.writerow(i)\n",
    "csvFile.close()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.3 抓取ig評論資料\n",
    "抓取熱門貼文(前九則)的內文，並儲存成CSV檔"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#搜尋各location內的連結\n",
    "URI='https://www.instagram.com'\n",
    "driver=webdriver.Chrome()\n",
    "driver.get(URI)\n",
    "\n",
    "#登入\n",
    "time.sleep(1)\n",
    "accounts = driver.find_element_by_xpath('//*[@name=\"username\"]')\n",
    "time.sleep(1)\n",
    "accounts.send_keys(\"帳號名稱\")\n",
    "time.sleep(1)\n",
    "pw = driver.find_element_by_xpath('//*[@name=\"password\"]')\n",
    "time.sleep(1)\n",
    "pw.send_keys(\"密碼\")\n",
    "login_btn = driver.find_element_by_xpath(\"//button[@class='sqdOP  L3NKy   y3zKF     ']\")\n",
    "login_btn.click()\n",
    "time.sleep(3)\n",
    "\n",
    "#是否自動登入：選擇稍後再說\n",
    "later_btn = driver.find_element_by_xpath(\"//button[@class='sqdOP yWX7d    y3zKF     ']\")\n",
    "later_btn.click()\n",
    "time.sleep(2)\n",
    "\n",
    "#是否開啟通知：選擇稍後再說\n",
    "later_btn_2 = driver.find_element_by_xpath(\"//button[@class='aOOlW   HoLwm ']\")\n",
    "later_btn_2.click()\n",
    "time.sleep(2)\n",
    "\n",
    "\n",
    "def search_for_post_links(location):\n",
    "    '''從location中抓出貼文連結'''\n",
    "    driver.get(location)\n",
    "    time.sleep(1)\n",
    "\n",
    "    #擷取連結\n",
    "    try:\n",
    "        a_list = driver.find_elements_by_xpath(\"//div[@class='v1Nh3 kIKUG  _bz0w']/a\")\n",
    "    except:\n",
    "        try:\n",
    "            time.sleep(5)\n",
    "            a_list = driver.find_elements_by_xpath(\"//div[@class='v1Nh3 kIKUG  _bz0w']/a\")\n",
    "        except:\n",
    "            return \"null\"\n",
    "    link_list = []\n",
    "    try: \n",
    "        for i in a_list:\n",
    "            link_list.append(i.get_attribute(\"href\"))\n",
    "    except:\n",
    "        link_list.append('null')\n",
    "    return link_list\n",
    "\n",
    "def search_for_details(a_post): \n",
    "    '''從post_link抓取內文'''\n",
    "    driver.get(a_post)\n",
    "    time.sleep(1)\n",
    "\n",
    "    text_list = []\n",
    "    #擷取連結\n",
    "    try:\n",
    "        a_span = driver.find_elements_by_xpath(\"//div[@class='C4VMK']/span\")\n",
    "    except:\n",
    "        try:\n",
    "            time.sleep(5)\n",
    "            a_span = driver.find_elements_by_xpath(\"//div[@class='C4VMK']/span\")\n",
    "        except:\n",
    "            return \"null\"\n",
    "    return a_span\n",
    "\n",
    "def preprocessing(text):\n",
    "    '''刪除不該出現的文字'''\n",
    "    sent = str(text)\n",
    "    sent = sent.replace(\"\\n\", \"\")\n",
    "    sent = sent.replace(\"<class 'list'>\", \"\")\n",
    "    sent = sent.replace(\"-\", \"\")\n",
    "    return sent\n",
    "\n",
    "\n",
    "#開啟IG location檔案\n",
    "with open('p2.csv', newline='') as csvFile:\n",
    "    rows = csv.reader(csvFile)\n",
    "    rows = list(rows)\n",
    "    location_list = []\n",
    "    for i in range(1, len(rows)):\n",
    "        place_id = rows[i][0]\n",
    "        ig_link = rows[i][2]\n",
    "        location_list.append([place_id, ig_link])\n",
    "csvFile.close()\n",
    "\n",
    "for a_loc in location_list:\n",
    "    if a_loc[1] == 'null':\n",
    "        continue\n",
    "    else:\n",
    "        links = search_for_post_links(a_loc[1])\n",
    "        f = open(str(a_loc[0]) + '.txt', 'w', newline='', errors = 'ignore')\n",
    "        #決定要抓幾篇\n",
    "        if len(links) <= 9: \n",
    "            posts_amount = len(links)\n",
    "        else:\n",
    "            posts_amount = 9\n",
    "\n",
    "        for i in range(posts_amount):\n",
    "            texts = search_for_details(links[i])\n",
    "            for j in texts:\n",
    "                test = j.text\n",
    "                f.write(preprocessing(test))\n",
    "        f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.4 資料清理與合併\n",
    "將原始的文字檔去除換行、表情符號等等，最後存成CSV FILE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#整併txt檔案成csv\n",
    "def preprocessing(text):\n",
    "    '''刪除不該出現的文字'''\n",
    "    sent = str(text)\n",
    "    sent = sent.replace(\"\\n\", \"\")\n",
    "    sent = sent.replace(\"<class 'list'>\", \"\")\n",
    "    sent = sent.replace(\"-\", \"\")\n",
    "    sent = sent.replace(\",\", \"\")\n",
    "    return sent\n",
    "\n",
    "#讀取IGlink檔案\n",
    "with open('wenzhou_IG_links.csv', newline='') as csvFile:\n",
    "    rows = csv.reader(csvFile)\n",
    "    rows = list(rows)\n",
    "    location_list = []\n",
    "    for i in range(1, len(rows)):\n",
    "        place_id = rows[i][0]\n",
    "        name = rows[i][1]\n",
    "        ig_link = rows[i][2]\n",
    "        location_list.append([place_id, name, ig_link])\n",
    "csvFile.close()\n",
    "\n",
    "#open新csv檔案\n",
    "fh1 = open('wenzhou_IG_content.csv', 'w', newline='', errors = 'ignore')\n",
    "writer = csv.writer(fh1)\n",
    "writer.writerow(['place_id', 'name', 'IG_link', 'content'])\n",
    "for a_restaurant in location_list:\n",
    "    new_list = []\n",
    "    new_list.extend(a_restaurant)\n",
    "    res_id = a_restaurant[0]\n",
    "    try:\n",
    "        fh_res = open(str(res_id) + '.txt', 'r', newline='', errors = 'ignore')\n",
    "        content = fh_res.read()\n",
    "        content = preprocessing(content)\n",
    "        content = str(content)\n",
    "        new_list.append(content)\n",
    "        fh_res.close()\n",
    "    except:\n",
    "        new_list.append('None')\n",
    "    writer.writerow(new_list)\n",
    "fh1.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 其他資料處理\n",
    "包含圖片轉檔、文字轉拼音兩個部分。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.1 圖片轉檔\n",
    "由於抓下來的時候是jpg flie，此處轉檔成PNG，以便前端使用，並進行壓縮檔案。\n",
    "這邊使用了PIL的套件，進行圖片處理。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "df = pd.read_csv('https://raw.githubusercontent.com/chhtwhc/FindFoodEatFood/main/TextMining.csv',usecols=[4])\n",
    "ids = np.array(df)\n",
    "id_list=ids.tolist()\n",
    "print(id_list)\n",
    "#轉檔\n",
    "for i in id_list:\n",
    "    try: \n",
    "        im1 = Image.open(r'food_old\\\\' + str(i[0]) + '.png')\n",
    "        im1.save(r'food\\\\' + str(i[0]) + '.png', optimize = True, quality = 15)\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.2 文字轉拼音\n",
    "這邊使用了pypinyin的套件，把漢字轉為拼音資料。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install pypinyin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "from pypinyin import pinyin\n",
    "\n",
    "#開啟檔案\n",
    "with open('gongguan_details.csv', newline='') as csvFile:\n",
    "    rows = csv.reader(csvFile)\n",
    "    rows = list(rows)\n",
    "    location_list = []\n",
    "    for i in range(1, len(rows)):\n",
    "        place_id = rows[i][2]\n",
    "        name = rows[i][3]\n",
    "        location_list.append([place_id, name])\n",
    "csvFile.close()\n",
    "\n",
    "def convert(name):\n",
    "    \"\"\"文字轉拼音\"\"\"    \n",
    "    alist = pinyin(name)\n",
    "    total_str = str()\n",
    "    for i in alist:\n",
    "        total_str += str(i[0])\n",
    "        total_str += \" \"\n",
    "    return total_str\n",
    "\n",
    "#寫入csv檔案\n",
    "with open('gongguan_pinyin.csv', 'w', newline='', encoding = 'utf-8', errors = 'ignore') as csvfile:\n",
    "    writer = csv.writer(csvfile)\n",
    "    writer.writerow(['place_id', 'chinese_name', 'pinyin'])\n",
    "    for i in location_list:\n",
    "        pin_name = convert(i[1])\n",
    "        i.append(pin_name)\n",
    "        writer.writerow(i)\n",
    "csvFile.close()\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyMyx/IheVe4dSM7+v1cFJ8W",
   "name": "神資作業.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}