ptt 查ip國家位置.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import time\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import csv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "取得今日文章列表...\n",
      "共 1018 篇文章\n",
      "取得前 100 篇文章 IP\n",
      "查詢 IP: [新聞] 明文規定陳抗禁止設攤 6/1韓國瑜凱道造勢\n",
      "來自 Taiwan    作者是 teramars\n",
      "查詢 IP: Re: [問卦] 全世界有哪些政黨是公開希望被敵國統治\n",
      "來自 Taiwan    作者是 AGIknight\n",
      "查詢 IP: [問卦] 男躲國小廁所！偷拍5女童脫褲子如廁\n",
      "來自 Germany    作者是 norvasc\n",
      "查詢 IP: [新聞] 東莞數百公交司機汽車站維權 討工齡賠償\n",
      "來自 United States    作者是 eeth\n",
      "查詢 IP: Re: [新聞] 打死3歲兒 死了扔山谷 狠父遭判9年半\n",
      "來自 Taiwan    作者是 cirin\n",
      "查詢 IP: [新聞] 駕駛突開車門！男未戴安全帽 遭後方機車\n",
      "來自 Taiwan    作者是 hw102050\n",
      "查詢 IP: [臉書] Emmy Hu\n",
      "來自 Taiwan    作者是 cycling\n",
      "查詢 IP: Re: [爆卦] 扁無罪卻被紅杉軍惡搞 扁:因為施明德恨我\n",
      "來自 Taiwan    作者是 EvilPrada\n",
      "查詢 IP: [問卦] 沒繳稅會怎樣?\n",
      "來自 Taiwan    作者是 ordonez\n",
      "查詢 IP: Re: [問卦] 毛澤東抓到竊聽器的時候怎麼不生氣\n",
      "來自 Taiwan    作者是 NTUSTNTU\n",
      "各國 IP 分布\n",
      "Taiwan 8\n",
      "Germany 1\n",
      "United States 1\n"
     ]
    }
   ],
   "source": [
    "\n",
    "PTT_URL = 'https://www.ptt.cc'\n",
    "API_KEY = '26266720e9aa08a4477f6ed04c1404ec'\n",
    "\n",
    "\n",
    "def get_web_page(url):\n",
    "    resp = requests.get(\n",
    "        url=url,\n",
    "        cookies={'over18': '1'}\n",
    "    )\n",
    "    if resp.status_code != 200:\n",
    "        print('Invalid url:', resp.url)\n",
    "        return None\n",
    "    else:\n",
    "        return resp.text\n",
    "\n",
    "\n",
    "def get_articles(dom, date):\n",
    "    soup = BeautifulSoup(dom, 'html5lib')\n",
    "\n",
    "    # 取得上一頁的連結\n",
    "    paging_div = soup.find('div', 'btn-group btn-group-paging')\n",
    "    prev_url = paging_div.find_all('a')[1]['href']\n",
    "\n",
    "    articles = []  # 儲存取得的文章資料\n",
    "    authortotal=[]\n",
    "    divs = soup.find_all('div', 'r-ent')\n",
    "    for d in divs:\n",
    "        if d.find('div', 'date').text.strip() == date:  # 發文日期正確\n",
    "            # 取得推文數\n",
    "            push_count = 0\n",
    "            push_str = d.find('div', 'nrec').text\n",
    "            if push_str:\n",
    "                try:\n",
    "                    push_count = int(push_str)  # 轉換字串為數字\n",
    "                except ValueError:\n",
    "                    # 若轉換失敗，可能是'爆'或 'X1', 'X2', ...\n",
    "                    # 若不是, 不做任何事，push_count 保持為 0\n",
    "                    if push_str == '爆':\n",
    "                        push_count = 99\n",
    "                    elif push_str.startswith('X'):\n",
    "                        push_count = -10\n",
    "\n",
    "            # 取得文章連結及標題\n",
    "            if d.find('a'):  # 有超連結，表示文章存在，未被刪除\n",
    "                href = d.find('a')['href']\n",
    "                title = d.find('a').text\n",
    "                author = d.find('div', 'author').text if d.find('div', 'author') else ''\n",
    "                articles.append({\n",
    "                    'title': title,\n",
    "                    'href': href,\n",
    "                    'push_count': push_count,\n",
    "                    'author': author\n",
    "                })\n",
    "                authortotal.append(author)\n",
    "    return articles, prev_url,authortotal\n",
    "\n",
    "\n",
    "def get_ip(dom):\n",
    "    # e.g., ※ 發信站: 批踢踢實業坊(ptt.cc), 來自: 27.52.6.175\n",
    "    pattern = '來自: \\d+\\.\\d+\\.\\d+\\.\\d+'\n",
    "    match = re.search(pattern, dom)\n",
    "    if match:\n",
    "        return match.group(0).replace('來自: ', '')\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "\n",
    "def get_country(ip):\n",
    "    if ip:\n",
    "        url = 'http://api.ipstack.com/{}?access_key={}'.format(ip, API_KEY)\n",
    "        data = requests.get(url).json()\n",
    "        country_name = data['country_name'] if data['country_name'] else None\n",
    "        return country_name\n",
    "    return None\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    print('取得今日文章列表...')\n",
    "    current_page = get_web_page(PTT_URL + '/bbs/Gossiping/index.html')\n",
    "    if current_page:\n",
    "        articles = []  # 全部的今日文章\n",
    "        author=[]\n",
    "        countryT=[]\n",
    "        title=[]\n",
    "        today = time.strftime('%m/%d').lstrip('0')  # 今天日期, 去掉開頭的 '0' 以符合 PTT 網站格式\n",
    "        current_articles, prev_url,authortotal = get_articles(current_page, today)  # 目前頁面的今日文章\n",
    "        while current_articles:  # 若目前頁面有今日文章則加入 articles，並回到上一頁繼續尋找是否有今日文章\n",
    "            articles += current_articles\n",
    "            current_page = get_web_page(PTT_URL + prev_url)\n",
    "            current_articles, prev_url,authortotal = get_articles(current_page, today)\n",
    "        print('共 %d 篇文章' % (len(articles)))\n",
    "\n",
    "        # 已取得文章列表，開始進入各文章尋找發文者 IP\n",
    "        print('取得前 100 篇文章 IP')\n",
    "        country_to_count = dict()\n",
    "        for article in articles[:100]:\n",
    "            print('查詢 IP:', article['title'])\n",
    "            page = get_web_page(PTT_URL + article['href'])\n",
    "            if page:\n",
    "                ip = get_ip(page)\n",
    "                country = get_country(ip)\n",
    "                if country in country_to_count.keys():\n",
    "                    country_to_count[country] += 1\n",
    "                else:\n",
    "                    country_to_count[country] = 1\n",
    "            print(\"來自\",country, end='')\n",
    "            print(\"   \",\"作者是\",article['author'])\n",
    "            author.append(article['author'])\n",
    "            title.append(article['title'])\n",
    "            countryT.append(country)\n",
    "        # 印出各國 IP 次數資訊\n",
    "        print('各國 IP 分布')\n",
    "        for k, v in country_to_count.items():\n",
    "            print(k, v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('output.csv', 'w', newline='') as csvfile:\n",
    "            writer = csv.writer(csvfile, delimiter=';') \n",
    "            writer.writerow(['文章標題', '作者', '國家'])\n",
    "            for i in range(len(author)):\n",
    "                writer.writerow([title[i],author[i],countryT[i]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}