daum-cafe-mobile-crawler-reply.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from bs4 import BeautifulSoup\n",
    "import sqlite3\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome('./driver/chromedriver')\n",
    "driver.get('https://logins.daum.net/accounts/signinform.do?url=http%3A%2F%2Fm.cafe.daum.net%2F_myCafe%3Fnull')\n",
    "\n",
    "time.sleep(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 수동으로 웹 브라우저에 직접 입력 후 건너뛰어도 됩니다. \n",
    "\n",
    "#driver.find_element_by_xpath(\"\"\"//*[@id=\"id\"]\"\"\").send_keys('') # id를 입력한다. \n",
    "#driver.find_element_by_xpath(\"\"\"//*[@id=\"inputPwd\"]\"\"\").send_keys('') # 패스워드를 입력한다. \n",
    "#driver.find_element_by_xpath(\"\"\"//*[@id=\"loginBtn\"]\"\"\").click() # 입력 버튼 클릭.\n",
    "#time.sleep(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "CAFE_NAME = 'ssaumjil' \n",
    "\n",
    "BOARD_NAME = 'Jntr' \n",
    "\n",
    "DB = CAFE_NAME + '_m_article.db'\n",
    "#DB에 저장하겠습니다. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DB 초기화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn = sqlite3.connect(DB)\n",
    "cur = conn.cursor()\n",
    "\n",
    "sql = 'CREATE TABLE IF NOT EXISTS \"' + BOARD_NAME + \\\n",
    "  '\"(\"num\" INTEGER NOT NULL UNIQUE, \"subject\" TEXT, ' \\\n",
    "  '\"nick\" TEXT, \"write_time\" TEXT, \"views\" INTEGER, ' \\\n",
    "  '\"url\" TEXT, \"contents\" TEXT, PRIMARY KEY(\"num\"));'\n",
    "\n",
    "cur.execute(sql)\n",
    "conn.commit()\n",
    "\n",
    "sql = 'CREATE TABLE IF NOT EXISTS \"' + BOARD_NAME + \\\n",
    "  '_cmt\" (\"cmt_num\" TEXT NOT NULL UNIQUE, \"cmt_reply\" INTEGER, ' \\\n",
    "  '\"cmt_writer\" TEXT, \"cmt_time\" TEXT, \"cmt_txt\" TEXT, PRIMARY KEY(\"cmt_num\"));'\n",
    "\n",
    "cur.execute(sql)\n",
    "conn.commit()\n",
    "\n",
    "conn.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('http://m.cafe.daum.net/%s/%s?boardType=' % (CAFE_NAME, BOARD_NAME))\n",
    "time.sleep(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "저장할 게시물 번호 + 엔터: 41\n"
     ]
    }
   ],
   "source": [
    "inp_num = input('저장할 게시물 번호 + 엔터: ')\n",
    "#inp_num = '41'\n",
    "num = int(inp_num)\n",
    "\n",
    "url = 'http://m.cafe.daum.net/%s/%s/%s' % (CAFE_NAME, BOARD_NAME, inp_num)\n",
    "driver.get(url)\n",
    "\n",
    "time.sleep(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = driver.page_source\n",
    "soup = BeautifulSoup(html, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "subject = soup.select_one('#mArticle > div.view_subject.\\#subject_area > h3')\n",
    "# repr(subject)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if subject is None:\n",
    "    print(url, '지워진 게시물입니다.')\n",
    "else:\n",
    "    subject = subject.get_text(strip=True)  \n",
    "    if soup.body.find('span', class_='txt_subject').find('span', class_='sr_only').get_text() == '작성자':\n",
    "        nick = soup.body.find('span', class_='sr_only').next_sibling\n",
    "    else:\n",
    "        nick = ''        \n",
    "    num_subject = soup.body.find_all('span', class_='num_subject')\n",
    "    write_time = num_subject[0].get_text()\n",
    "    views = num_subject[1].get_text()\n",
    "    print(num, subject, nick, write_time, views, url)\n",
    "\n",
    "    contents = soup.body.find('div', id='article').get_text('\\n', strip=True)\n",
    "    print(contents)\n",
    "    \n",
    "    # DB 저장\n",
    "    \n",
    "    conn = sqlite3.connect(DB)\n",
    "    cur = conn.cursor()\n",
    "    sql = \"replace into %s(num,subject,nick,write_time,views,url,contents) values (?,?,?,?,?,?,?)\" % BOARD_NAME\n",
    "    cur.execute(sql, (num, subject, nick, write_time, views, url, contents))\n",
    "    conn.commit()\n",
    "    conn.close()\n",
    "    \n",
    "    # 댓글\n",
    "    \n",
    "    cmt_total = soup.body.find('article', id='mArticle').find('span', class_=\"num_total\").get_text()\n",
    "    print('댓글 개수 :', cmt_total)\n",
    "    \n",
    "    # 댓글 페이지로 이동\n",
    "\n",
    "    driver.get('http://m.cafe.daum.net/{0}/{1}/{2}/comments'.format(CAFE_NAME,BOARD_NAME,inp_num))\n",
    "    time.sleep(1)\n",
    "    \n",
    "    # 페이지 숫자를 확인하기 위해 캡쳐\n",
    "\n",
    "    html = driver.page_source\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    \n",
    "    # 페이지 숫자 확인\n",
    "    \n",
    "    cmt_page_max = int(soup.body.find('span', id=\"pagingNav\").find_all('span', class_=\"num_page\").pop().get_text())\n",
    "    print('cmt_page_max', cmt_page_max)\n",
    "    \n",
    "    # 페이지를 하나씩 넘기면서 캡쳐.. \n",
    "    \n",
    "    for i in range(cmt_page_max):\n",
    "        j = cmt_page_max-i\n",
    "        \n",
    "        html = driver.page_source\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "        cmt_all = soup.body.find('ul' ,id=\"commentList\").find_all('li')\n",
    "        \n",
    "        for k in cmt_all:\n",
    "            if k.div.span.find_all('span', class_=\"txt_bar\"):\n",
    "                if k['class'] == ['reply_on']:\n",
    "                    cmt_reply = True\n",
    "                    print('ㄴ', end='')\n",
    "                else:\n",
    "                    cmt_reply = False\n",
    "\n",
    "                cmt_num = inp_num +'_'+ k['id'].split('comment_')[1]\n",
    "                cmt_writer = k.div.find('span', class_='sr_only').next_sibling\n",
    "                cmt_time = k.div.span.find('span', class_=\"num_info\").get_text()\n",
    "                cmt_txt = k.div.find('span', class_='txt_detail').get_text(strip=True)\n",
    "                print(cmt_num, '[', cmt_writer, ']', cmt_time, cmt_txt)\n",
    "                \n",
    "                conn = sqlite3.connect(DB)\n",
    "                cur = conn.cursor()\n",
    "                sql = \"replace into '%s_cmt'(cmt_num,cmt_reply,cmt_writer,cmt_time,cmt_txt) values (?,?,?,?,?)\" % BOARD_NAME\n",
    "                cur.execute(sql, (cmt_num,cmt_reply,cmt_writer,cmt_time,cmt_txt))\n",
    "                conn.commit()\n",
    "                conn.close()\n",
    "                \n",
    "        if j > 1:\n",
    "            driver.get('http://m.cafe.daum.net/{0}/{1}/{2}/comments?prev_page={3}&mode=regular&cdepth={4}&page={5}'.format(CAFE_NAME,BOARD_NAME,inp_num,j,'0002100000',(j-1)))\n",
    "            time.sleep(2)    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.close()\n",
    "# 브라우저를 닫습니다. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}