06. beautifulsoup의 parsing하는 여러가지 함수 사용.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.beautifulsoup의 parsing하는 여러 가지 함수 사용"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 select 사용하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n<ol>\\n    <li>NEVER - 국민의 아들</li> \\n    <li>SIGNAL - TWICE</li> \\n    <li>LONELY - 씨스타</li> \\n    <li>I LUV IT - PSY</li> \\n    <li>New Face - PSY</li>\\n</ol>\\n'"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup \n",
    "\n",
    "html = '''\n",
    "<ol>\n",
    "    <li>NEVER - 국민의 아들</li> \n",
    "    <li>SIGNAL - TWICE</li> \n",
    "    <li>LONELY - 씨스타</li> \n",
    "    <li>I LUV IT - PSY</li> \n",
    "    <li>New Face - PSY</li>\n",
    "</ol>\n",
    "'''\n",
    "\n",
    "html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<li class=\"first\" id=\"na\">NEVER - 국민의 아들</li>, <li class=\"first\" id=\"tw\">SIGNAL - TWICE</li>, <li class=\"second\" id=\"lo\">LONELY - 씨스타</li>, <li class=\"second\" id=\"lu\">I LUV IT - PSY</li>, <li>New Face - PSY</li>]\n",
      "<class 'list'>\n",
      "<li class=\"first\" id=\"tw\">SIGNAL - TWICE</li>\n",
      "<li class=\"first\" id=\"na\">NEVER - 국민의 아들</li>\n",
      "<class 'bs4.element.Tag'>\n"
     ]
    }
   ],
   "source": [
    "html = '''\n",
    "<ol>\n",
    "    <li id='na' class='first'>NEVER - 국민의 아들</li> \n",
    "    <li id='tw' class='first'>SIGNAL - TWICE</li> \n",
    "    <li id='lo' class='second'>LONELY - 씨스타</li> \n",
    "    <li id='lu' class='second'>I LUV IT - PSY</li> \n",
    "    <li>New Face - PSY</li>\n",
    "</ol>\n",
    "'''\n",
    "\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "print(soup.select('li'))\n",
    "print(type(soup.select('li')))\n",
    "print(soup.select('li')[1])\n",
    "print(soup.select_one('li'))\n",
    "print(type(soup.select_one('li')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NEVER - 국민의 아들\n",
      "SIGNAL - TWICE\n"
     ]
    }
   ],
   "source": [
    "# tag명이 li 이고, class 이름이  'first'인 tag를 가져오기\n",
    "for tag in soup.select('li.first'):\n",
    "    print(tag.string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SIGNAL - TWICE\n",
      "SIGNAL - TWICE\n"
     ]
    }
   ],
   "source": [
    "# tag의 id값이 'tw'인 tag 가져오기\n",
    "print(soup.select_one('#tw').text)\n",
    "print(soup.select('#tw')[0].string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'New Face - PSY'"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# li tag 중에서 5번째 li tag 가져오기\n",
    "soup.select_one('li:nth-of-type(5)').text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 fine(), find_all() Method 사용하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<li class=\"first\" id=\"na\">NEVER - 국민의 아들</li>, <li class=\"first\" id=\"tw\">SIGNAL - TWICE</li>, <li class=\"second\" id=\"lo\">LONELY - 씨스타</li>, <li class=\"second\" id=\"lu\">I LUV IT - PSY</li>, <li>New Face - PSY</li>]\n",
      "<class 'bs4.element.ResultSet'>\n",
      "LONELY - 씨스타\n",
      "I LUV IT - PSY\n"
     ]
    }
   ],
   "source": [
    "print(soup.find_all('li'))\n",
    "print(type(soup.find_all('li')))\n",
    "\n",
    "# tag명이 li이고, class 이름이 'second'인 tag를 가져오기\n",
    "for tag in soup.find_all('li', attrs={'class':'second'}):\n",
    "    print(tag.string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LONELY - 씨스타\n"
     ]
    }
   ],
   "source": [
    "print(soup.find('li', attrs={'class':'second', 'id':'lo'}).string)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. 기상청 날씨 데이터 파싱하기 (find, find_all 사용)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "url = \"http://www.kma.go.kr/weather/forecast/mid-term-rss3.jsp\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![기상청 데이터](./images/17.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = requests.get(url).text\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "# soup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![네이버 웹툰](./images/18.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A02 2018-06-17 00:00 보통\n",
      "A02 2018-06-17 12:00 보통\n",
      "A02 2018-06-18 00:00 보통\n",
      "A02 2018-06-18 12:00 보통\n",
      "A02 2018-06-19 00:00 보통\n",
      "A02 2018-06-19 12:00 보통\n",
      "A02 2018-06-20 00:00 보통\n",
      "A02 2018-06-20 12:00 보통\n",
      "A02 2018-06-21 00:00 보통\n",
      "A02 2018-06-21 12:00 보통\n",
      "A01 2018-06-22 00:00 보통\n",
      "A01 2018-06-23 00:00 보통\n",
      "A01 2018-06-24 00:00 보통\n"
     ]
    }
   ],
   "source": [
    "title = soup.find('header').find('title').text # 부분적인 값 가져 오기 \n",
    "datas = soup.find('location', attrs={'wl_ver':'3'}).find_all('data')\n",
    "for data in datas:\n",
    "    print(data.find('mode').string + ' ' + data.find('tmef').string + ' '+ data.find('reliability').text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.Melon 100곡 데이터 파싱하기 (select, select_on 사용)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 아래 3가지 정보를 알고 시작하자"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![멜론 사이트](./images/19.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![앨범 정보](./images/20.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### User-Agent (안 넣어 주면 값을 가져 오지 못한다.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![user-id 값](./images/21.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = { # 메론 사이트에 꼭들어 가야 하는 내용\n",
    "    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.melon.com/chart/day/index.htm'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = requests.get(url, headers=headers).text\n",
    "soup = BeautifulSoup(html, 'html.parser')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![긁어온 내용 처리 하기](./images/23.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "# song list 가져오기\n",
    "song_list = soup.select('#tb_list tr .wrap_song_info a[href*=playSong]')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### song list 만 가져 와서 정리 한 내용"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![정리한 내용 보여 주기](./images/24.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 정규 표현식 확인 가능한 Site : https://regexr.com/  뒤쪽 Song Id를 가져 오기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![정규표현식으로 처리 하기](./images/22.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 여행 https://www.melon.com/song/detail.htm?songId=31093710\n",
      "2 FAKE LOVE https://www.melon.com/song/detail.htm?songId=31085237\n",
      "3 지나오다 https://www.melon.com/song/detail.htm?songId=30699142\n",
      "4 밤 (Time for the moon night) https://www.melon.com/song/detail.htm?songId=31052421\n",
      "5 주지마 https://www.melon.com/song/detail.htm?songId=31035061\n",
      "6 빙글뱅글 (Bingle Bangle) https://www.melon.com/song/detail.htm?songId=31098083\n",
      "7 You https://www.melon.com/song/detail.htm?songId=30849733\n",
      "8 켜줘 (Light) https://www.melon.com/song/detail.htm?songId=31113240\n",
      "9 모든 날, 모든 순간 (Every day, Every Moment) https://www.melon.com/song/detail.htm?songId=30962526\n",
      "10 사랑을 했다 (LOVE SCENARIO) https://www.melon.com/song/detail.htm?songId=30859584\n",
      "11 What is Love? https://www.melon.com/song/detail.htm?songId=31006730\n",
      "12 별이 빛나는 밤 https://www.melon.com/song/detail.htm?songId=30937275\n",
      "13 캥거루 (Kangaroo) (Prod. ZICO) https://www.melon.com/song/detail.htm?songId=31113241\n",
      "14 붕붕 (Feat. 식케이) (Prod. GroovyRoom) https://www.melon.com/song/detail.htm?songId=31019584\n",
      "15 My Way https://www.melon.com/song/detail.htm?songId=30721801\n",
      "16 뿜뿜 https://www.melon.com/song/detail.htm?songId=30816860\n",
      "17 그날처럼 https://www.melon.com/song/detail.htm?songId=30755375\n",
      "18 LATATA https://www.melon.com/song/detail.htm?songId=31055324\n",
      "19 전하지 못한 진심 (Feat. Steve Aoki) https://www.melon.com/song/detail.htm?songId=31085238\n",
      "20 DNA https://www.melon.com/song/detail.htm?songId=30637982\n",
      "21 그때 헤어지면 돼 https://www.melon.com/song/detail.htm?songId=30884950\n",
      "22 HandClap https://www.melon.com/song/detail.htm?songId=8111603\n",
      "23 소나기 (Feat. 10cm) https://www.melon.com/song/detail.htm?songId=30970444\n",
      "24 모래시계 (Prod. 헤이즈) https://www.melon.com/song/detail.htm?songId=31113243\n",
      "25 봄날 https://www.melon.com/song/detail.htm?songId=30244931\n",
      "26 Roller Coaster https://www.melon.com/song/detail.htm?songId=30844118\n",
      "27 잊을만하면 https://www.melon.com/song/detail.htm?songId=31056799\n",
      "28 선물 https://www.melon.com/song/detail.htm?songId=30514366\n",
      "29 넋두리 https://www.melon.com/song/detail.htm?songId=9642570\n",
      "30 Anpanman https://www.melon.com/song/detail.htm?songId=31085244\n",
      "31 꽃 길 https://www.melon.com/song/detail.htm?songId=30948698\n",
      "32 Havana (Feat. Young Thug) https://www.melon.com/song/detail.htm?songId=30567963\n",
      "33 BOOMERANG (부메랑) https://www.melon.com/song/detail.htm?songId=30960341\n",
      "34 Shape of You https://www.melon.com/song/detail.htm?songId=30188113\n",
      "35 욕심 https://www.melon.com/song/detail.htm?songId=31046548\n",
      "36 EVERYDAY https://www.melon.com/song/detail.htm?songId=30997649\n",
      "37 가을 안부 https://www.melon.com/song/detail.htm?songId=30669593\n",
      "38 열애중 https://www.melon.com/song/detail.htm?songId=31062863\n",
      "39 에너제틱 (Energetic) https://www.melon.com/song/detail.htm?songId=30568338\n",
      "40 바코드 (Prod. GroovyRoom) https://www.melon.com/song/detail.htm?songId=30989550\n",
      "41 Airplane pt.2 https://www.melon.com/song/detail.htm?songId=31085243\n",
      "42 미안해 https://www.melon.com/song/detail.htm?songId=30809895\n",
      "43 이 별 https://www.melon.com/song/detail.htm?songId=30637776\n",
      "44 영원+1 (Prod. NELL) https://www.melon.com/song/detail.htm?songId=31113242\n",
      "45 빛나리 https://www.melon.com/song/detail.htm?songId=30992157\n",
      "46 Beautiful https://www.melon.com/song/detail.htm?songId=30725482\n",
      "47 이 밤의 끝 https://www.melon.com/song/detail.htm?songId=31107239\n",
      "48 밤편지 https://www.melon.com/song/detail.htm?songId=30314784\n",
      "49 비도 오고 그래서 (Feat. 신용재) https://www.melon.com/song/detail.htm?songId=30492279\n",
      "50 우리 둘만 아는 https://www.melon.com/song/detail.htm?songId=30702937\n",
      "51 The Feeling https://www.melon.com/song/detail.htm?songId=31123630\n",
      "52 낙원 https://www.melon.com/song/detail.htm?songId=31085240\n",
      "53 밤이 되니까 https://www.melon.com/song/detail.htm?songId=30659630\n",
      "54 그리워하다 https://www.melon.com/song/detail.htm?songId=30672529\n",
      "55 Magic Shop https://www.melon.com/song/detail.htm?songId=31085242\n",
      "56 Jenga (Feat. Gaeko) https://www.melon.com/song/detail.htm?songId=30939452\n",
      "57 Love Maze https://www.melon.com/song/detail.htm?songId=31085241\n",
      "58 11 (Prod. 다이나믹듀오) https://www.melon.com/song/detail.htm?songId=31113244\n",
      "59 Bad Boy https://www.melon.com/song/detail.htm?songId=30864993\n",
      "60 Thunder https://www.melon.com/song/detail.htm?songId=30393293\n",
      "61 좋니 https://www.melon.com/song/detail.htm?songId=30486509\n",
      "62 약속해요 (I.P.U.) https://www.melon.com/song/detail.htm?songId=30930312\n",
      "63 134340 https://www.melon.com/song/detail.htm?songId=31085239\n",
      "64 비 그리고 너 https://www.melon.com/song/detail.htm?songId=31126547\n",
      "65 눈 떠보니 이별이더라 https://www.melon.com/song/detail.htm?songId=30690674\n",
      "66 all of my life https://www.melon.com/song/detail.htm?songId=30550388\n",
      "67 Lonely https://www.melon.com/song/detail.htm?songId=31093714\n",
      "68 눈사람 https://www.melon.com/song/detail.htm?songId=30877002\n",
      "69 불행 (Feat. 빈첸 (VINXEN)) https://www.melon.com/song/detail.htm?songId=31126139\n",
      "70 북 (Prod. SLO) https://www.melon.com/song/detail.htm?songId=30957524\n",
      "71 REALLY REALLY https://www.melon.com/song/detail.htm?songId=30341745\n",
      "72 instagram https://www.melon.com/song/detail.htm?songId=30806580\n",
      "73 빨간 맛 (Red Flavor) https://www.melon.com/song/detail.htm?songId=30512671\n",
      "74 시차 (We Are) (Feat. 로꼬 & GRAY) https://www.melon.com/song/detail.htm?songId=30613202\n",
      "75 Heart Shaker https://www.melon.com/song/detail.htm?songId=30781481\n",
      "76 So What https://www.melon.com/song/detail.htm?songId=31085245\n",
      "77 花요일 (Blooming Day) https://www.melon.com/song/detail.htm?songId=31009483\n",
      "78 데리러 가 (Good Evening) https://www.melon.com/song/detail.htm?songId=31097925\n",
      "79 비행운 https://www.melon.com/song/detail.htm?songId=30086173\n",
      "80 YOUTH https://www.melon.com/song/detail.htm?songId=7928705\n",
      "81 전혀 (Feat. 우원재) (Prod. GroovyRoom) https://www.melon.com/song/detail.htm?songId=31019586\n",
      "82 There`s Nothing Holdin` Me Back https://www.melon.com/song/detail.htm?songId=30380953\n",
      "83 LIKEY https://www.melon.com/song/detail.htm?songId=30700708\n",
      "84 Playlist https://www.melon.com/song/detail.htm?songId=31126160\n",
      "85 어른 https://www.melon.com/song/detail.htm?songId=30985406\n",
      "86 Intro : Singularity https://www.melon.com/song/detail.htm?songId=31085236\n",
      "87 마지막처럼 https://www.melon.com/song/detail.htm?songId=30481578\n",
      "88 어린 왕자 (Prod. Godic) https://www.melon.com/song/detail.htm?songId=30974156\n",
      "89 Outro : Tear https://www.melon.com/song/detail.htm?songId=31085246\n",
      "90 바람사람 https://www.melon.com/song/detail.htm?songId=31093709\n",
      "91 Universe https://www.melon.com/song/detail.htm?songId=30806536\n",
      "92 #첫사랑 https://www.melon.com/song/detail.htm?songId=30829858\n",
      "93 결혼 https://www.melon.com/song/detail.htm?songId=30689468\n",
      "94 다시, 우리 https://www.melon.com/song/detail.htm?songId=31093282\n",
      "95 FAKE LOVE (Rocking Vibe Mix) https://www.melon.com/song/detail.htm?songId=31113142\n",
      "96 첫눈처럼 너에게 가겠다 https://www.melon.com/song/detail.htm?songId=30190630\n",
      "97 비 https://www.melon.com/song/detail.htm?songId=8235260\n",
      "98 노력 https://www.melon.com/song/detail.htm?songId=30099927\n",
      "99 썸 탈꺼야 https://www.melon.com/song/detail.htm?songId=30657307\n",
      "100 짙어져 https://www.melon.com/song/detail.htm?songId=30667988\n"
     ]
    }
   ],
   "source": [
    "for idx, song in enumerate(song_list, 1):\n",
    "    title = song.text\n",
    "    link = song['href']\n",
    "    matched = re.search(r\"(\\d+)\\);\", link) # 정규 표현식 처리 \n",
    "    \n",
    "    if matched:\n",
    "        songId = matched.group(1)\n",
    "        songUrl = 'https://www.melon.com/song/detail.htm?songId='+ songId # song 정보 가져 오기\n",
    "        print(idx, title, songUrl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}