愛食記資料蒐集.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install requests\n",
    "!pip install bs4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### 利用愛食記,查詢三大餐飲集團旗下各品牌餐廳的相關資訊 ####\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import json\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "### 編寫爬蟲架構--根據搜尋的關鍵字,每次爬取3頁(爬太多頁就會搜到很多錯的)的分店大標題和其內容(一頁最多顯示15個大標題)\n",
    "def crawler(company,restaurant):\n",
    "  headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36'}\n",
    "  final_list = [] \n",
    "\n",
    "  for i in range(3):\n",
    "    try:    # 有的餐廳比較冷門會查不到半篇\n",
    "      url = f'https://ifoodie.tw/explore/list/{restaurant}?page={i+1}'\n",
    "      res = requests.get(url,headers=headers)\n",
    "      soup = BeautifulSoup(res.text,'html.parser')\n",
    "      main_tag = soup.select('div[class=\"jsx-558691085 info-rows\"]')  \n",
    "      \n",
    "      for tag in main_tag:\n",
    "        main_article_title = tag.find('div',{'class':\"jsx-558691085 title\"}).text.split(\".\")[1].strip()   # 分店大標題\n",
    "        main_article_url = \"https://ifoodie.tw\"+tag.find('div',{'class':\"jsx-558691085 title\"}).find('a').get('href')  # 大標題連結(內有評論和相關食記)\n",
    "        main_article_score = tag.find('div',{'class':\"jsx-1207467136 text\"}).text.strip()   # 分店評分\n",
    "        main_article_reviewcount = tag.find('a',{'class':\"jsx-558691085 review-count\"}).text.split(\"則\")[0].split(\"(\")[1].strip()  # 分店評論數\n",
    "        main_article_category = tag.findAll('a',{'class':\"jsx-558691085 category\"})[1].text.strip()  # 分店菜系\n",
    "\n",
    "        ## 針對每個大標題,進入連結取得所有評論及相關食記的內容\n",
    "        # 發現評論和內容是以動態網頁呈現,所以直接從network找尋API,並找到URL的規律\n",
    "        if restaurant in main_article_title:   # 有些查詢到的分店跟關鍵字根本無關,先過濾掉\n",
    "          # 從大標題的URL萃取出API有用到的article_id\n",
    "          article_id = main_article_url.split(\"restaurant/\")[1].split(\"-\")[0]  \n",
    "\n",
    "          ## 萃取評論文字\n",
    "          review_api = f'https://ifoodie.tw/api/checkin/?restaurant_id={article_id}&offset=0&limit=100'\n",
    "          review_dict = json.loads(requests.get(review_api).text)\n",
    "          review_content = \"\"\n",
    "\n",
    "          for row in range(len(review_dict[\"response\"])):\n",
    "            review = review_dict[\"response\"][row][\"message\"] \n",
    "            # 有些評論是食記,先跳過\n",
    "            if review_dict[\"response\"][row][\"user\"][\"certified\"] == False:   \n",
    "                review_content = review_content + review + \"\\n-------------------------------------------\\n\"\n",
    "            else:\n",
    "              continue\n",
    "\n",
    "          ## 萃取食記URL及文章內容文字\n",
    "          article_api = f'https://ifoodie.tw/api/restaurant/{article_id}/blogs/?offset=0&limit=100'          \n",
    "          article_dict = json.loads(requests.get(article_api).text)\n",
    "          article_info_list = []\n",
    "\n",
    "          for row2 in range(len(article_dict[\"response\"])):\n",
    "            # 找到每篇食記的URL\n",
    "            blog_id = article_dict[\"response\"][row2][\"stat\"][\"blog_id\"]\n",
    "            article_url = f'https://ifoodie.tw/post/{blog_id}'\n",
    "            \n",
    "            try:     # 有的食記是外部網站,不適用這種格式的URL,先略過\n",
    "              # 進入連結並爬取每個食記的文章內容\n",
    "              res2 = requests.get(article_url)\n",
    "              soup2 = BeautifulSoup(res2.text,'html.parser')\n",
    "              article_time = soup2.find('div',{'class':\"jsx-3054895173 publish-date\"}).text.split(\"：\")[1].strip()\n",
    "              content = soup2.findAll('p',{'class':\"jsx-472180199\"})\n",
    "              article_title = soup2.find('h1',{'class':\"jsx-3054895173 post-title\"}).text.strip()\n",
    "              article_content = article_title + \"-----------\"\n",
    "              for texttag in content :\n",
    "                  article_content = article_content + texttag.text.strip() + \",\"\n",
    "\n",
    "              # 把文章資訊統整在一個list內\n",
    "              article_info_list.append(\n",
    "                  {'文章連結' : article_url,\n",
    "                  '發布時間' : article_time,\n",
    "                  '內容' : article_content}\n",
    "                  )\n",
    "\n",
    "            except:\n",
    "              pass\n",
    "\n",
    "          ## 將所有資訊整理進一個list裡面,以便之後轉換為df\n",
    "          final_list.append(\n",
    "            {'來源網站': \"愛食記\",\n",
    "            '集團': company,\n",
    "            '品牌': restaurant,\n",
    "            '標題': main_article_title,\n",
    "            '連結' : main_article_url,\n",
    "            '菜系': main_article_category,\n",
    "            '評分': main_article_score,\n",
    "            '留言數': main_article_reviewcount,\n",
    "            '文章內容': review_content, \n",
    "            '食記資訊': article_info_list}\n",
    "            )\n",
    "\n",
    "        else:\n",
    "          continue\n",
    "\n",
    "    except:\n",
    "      pass\n",
    "\n",
    "  ## 將資料轉換成表格呈現\n",
    "  df = pd.DataFrame(final_list)\n",
    "  try:   # 有些冷門餐廳查不到資料,會無法生成df\n",
    "    df.to_csv(f\"C:\\\\Users\\\\user\\\\Desktop\\\\ifoodie\\\\{company}\\\\{restaurant}.csv\",index=0,encoding='utf-8-sig',header=['來源網站','集團','品牌','標題','連結','菜系','評分','留言數','文章內容','食記資訊'])    # 路徑請視自身環境修改\n",
    "  except:\n",
    "    pass\n",
    "\n",
    "\n",
    "### 輸入關鍵字開始爬蟲\n",
    "restaurant_list = [{\"company\":\"王品集團\",\"restaurant\": [\"王品牛排\",\"西堤\",\"陶板屋\",\"原燒\",\"聚北海道鍋物\",\"藝奇\",\"夏慕尼\",\"品田牧場\",\"石二鍋\",\"hot 7\",\"莆田\",\"青花驕\",\"享鴨\",\"丰禾\",\"樂越\",\"12MINI\",\"THE WANG\",\"和牛涮\",\"尬鍋\",\"肉次方\"]},\n",
    "                   {\"company\":\"瓦城泰統\",\"restaurant\": ['瓦城','非常泰','1010湘','大心','時時香','月月THAI','YABI KITCHEN']},\n",
    "                   {\"company\":\"漢來美食\",\"restaurant\": ['漢來海港','漢來名人坊','漢來蔬食','漢來軒','翠園粵菜','五梅先生','安那居','上菜','福園台菜','弁慶','精緻海鮮火鍋','糕餅小舖','大廳酒廊','漢來咖啡廳','Pavo','焰·鐵板燒']}]\n",
    "\n",
    "for n in range(3):\n",
    "  # 為各個集團創建資料夾存放CSV檔(路徑請視自身環境修改)\n",
    "  if not os.path.exists(f\"C:\\\\Users\\\\user\\\\Desktop\\\\ifoodie\\\\{restaurant_list[n]['company']}\"):\n",
    "    os.mkdir(f\"C:\\\\Users\\\\user\\\\Desktop\\\\ifoodie\\\\{restaurant_list[n]['company']}\")\n",
    "\n",
    "  for restaurant in range(len(restaurant_list[n][\"restaurant\"])):\n",
    "    crawler(restaurant_list[n][\"company\"],restaurant_list[n][\"restaurant\"][restaurant])    "
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "247ab06e135bb35fa78c5eff31b2a9a0050dcb5fb773c2631d2a29ac689eeccb"
  },
  "kernelspec": {
   "display_name": "Python 3.9.6 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}