bs4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "解析库\n",
    "html.parser,lxml,xml,html5lib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html lang=\"en\">\n",
      " <head>\n",
      "  <meta charset=\"utf-8\"/>\n",
      "  <title>\n",
      "   Title\n",
      "  </title>\n",
      " </head>\n",
      "</html>\n",
      "Title\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 基本使用 不完整的html也能解析\n",
    "from bs4 import BeautifulSoup\n",
    "html='''\n",
    "<!DOCTYPE html>\n",
    "<html lang=\"en\">\n",
    "<head>\n",
    "    <meta charset=\"UTF-8\">\n",
    "    <title>Title\n",
    "'''\n",
    "soup=BeautifulSoup(html,'lxml')\n",
    "print(soup.prettify())\n",
    "print(soup.title.string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<title>Title\n",
      "    </title>\n",
      "<class 'bs4.element.Tag'>\n",
      "<head>\n",
      "<meta charset=\"utf-8\"/>\n",
      "<title>Title\n",
      "    </title></head>\n",
      "<p name=\"pname\">afa\n",
      "    <a>a1</a>\n",
      "<a>a2</a>\n",
      "</p>\n",
      "head\n",
      "pname\n",
      "pname\n",
      "None\n",
      "Title\n",
      "    \n",
      "子节点和子孙节点\n",
      "['afa\\n    ', <a>a1</a>, '\\n', <a>a2</a>, '\\n']\n",
      "<list_iterator object at 0x11209d198>\n",
      "0 afa\n",
      "    \n",
      "1 <a>a1</a>\n",
      "2 \n",
      "\n",
      "3 <a>a2</a>\n",
      "4 \n",
      "\n",
      "-------\n",
      "<generator object Tag.descendants at 0x11208f390>\n",
      "0 afa\n",
      "    \n",
      "1 <a>a1</a>\n",
      "2 a1\n",
      "3 \n",
      "\n",
      "4 <a>a2</a>\n",
      "5 a2\n",
      "6 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 标签选择器\n",
    "from bs4 import BeautifulSoup\n",
    "html='''\n",
    "<!DOCTYPE html>\n",
    "<html lang=\"en\">\n",
    "<head>\n",
    "    <meta charset=\"UTF-8\">\n",
    "    <title>Title\n",
    "    <body>\n",
    "    body test\n",
    "    <p name='pname'>afa\n",
    "    <a>a1</a>\n",
    "    <a>a2</a>\n",
    "    </p>\n",
    "     <ul>\n",
    "     <li id=\"list-1\">a</li>\n",
    "     <li name=\"list2\">b</li>\n",
    "     <li class='abc'>c</li>\n",
    "     <li class='abc def'>abc</li>\n",
    "     </ul>\n",
    "    </body>\n",
    "'''\n",
    "soup=BeautifulSoup(html,'lxml')\n",
    "print(soup.title)\n",
    "print(type(soup.title))\n",
    "print(soup.head)\n",
    "print(soup.p) \n",
    "#获取名称\n",
    "print(soup.head.name)\n",
    "#获取属性\n",
    "print(soup.p.attrs['name'])\n",
    "print(soup.p['name'])\n",
    "#获取内容\n",
    "print(soup.p.string)\n",
    "#潜套选择\n",
    "print(soup.head.title.string)\n",
    "#子节点和子孙节点\n",
    "print('子节点和子孙节点')\n",
    "print(soup.p.contents)\n",
    "print(soup.p.children)\n",
    "for i,child in enumerate(soup.p.children):\n",
    "    print(i,child)\n",
    "print('-------')\n",
    "print(soup.p.descendants)\n",
    "for i,child in enumerate(soup.p.descendants):\n",
    "    print(i,child)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p name=\"pname\">afa\n",
      "    <a>a1</a>\n",
      "<a>a2</a>\n",
      "</p>\n",
      "<generator object PageElement.parents at 0x11208f318>\n"
     ]
    }
   ],
   "source": [
    "#父节点\n",
    "print(soup.a.parent)\n",
    "print(soup.a.parents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, '\\n'), (1, <a>a2</a>), (2, '\\n')]\n",
      "[(0, 'afa\\n    ')]\n"
     ]
    }
   ],
   "source": [
    "#兄弟节点\n",
    "print(list(enumerate(soup.a.next_siblings)))\n",
    "print(list(enumerate(soup.a.previous_siblings)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<ul>\n",
      "<li id=\"list-1\">a</li>\n",
      "<li name=\"list2\">b</li>\n",
      "<li class=\"abc\">c</li>\n",
      "<li class=\"abc def\">abc</li>\n",
      "</ul>]\n",
      "<class 'bs4.element.Tag'>\n"
     ]
    }
   ],
   "source": [
    "#标准选择器\n",
    "print(soup.find_all('ul'))\n",
    "print(type(soup.find_all('ul')[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<li id=\"list-1\">a</li>, <li name=\"list2\">b</li>, <li class=\"abc\">c</li>, <li class=\"abc def\">abc</li>]\n"
     ]
    }
   ],
   "source": [
    "for ul in soup.find_all('ul'):\n",
    "    print(ul.find_all('li'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<li id=\"list-1\">a</li>]\n",
      "[<li name=\"list2\">b</li>]\n"
     ]
    }
   ],
   "source": [
    "# attrs\n",
    "print(soup.find_all(attrs={'id':'list-1'}))\n",
    "print(soup.find_all(attrs={'name':'list2'}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<li id=\"list-1\">a</li>]\n",
      "[<li class=\"abc\">c</li>, <li class=\"abc def\">abc</li>]\n",
      "['abc']\n"
     ]
    }
   ],
   "source": [
    "# text  id class_\n",
    "print(soup.find_all(id='list-1'))\n",
    "print(soup.find_all(class_='abc'))\n",
    "print(soup.find_all(text='abc'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<ul>\n",
      "<li id=\"list-1\">a</li>\n",
      "<li name=\"list2\">b</li>\n",
      "<li class=\"abc\">c</li>\n",
      "<li class=\"abc def\">abc</li>\n",
      "</ul>\n"
     ]
    }
   ],
   "source": [
    "print(soup.find('ul'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup.find_parents() #返回所有祖先节点\n",
    "soup.find_parent() #返回直接父节点\n",
    "soup.find_next_siblings() #返回后面所有兄弟节点\n",
    "soup.find_next_sibling() #返回后面第一个兄弟节点\n",
    "soup.find_previous_siblings() #返回前面所有兄弟节点\n",
    "soup.find_previous_sibling() #返回前面第一个兄弟节点\n",
    "soup.find_all_next() #返回后面所有符合条件节点\n",
    "soup.find_next() #返回第一个符合条件的节点\n",
    "soup.find_all_previous() #返回前面所有符合条件节点\n",
    "soup.find_previous() #返回前面第一个符合条件节点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<li class=\"abc\">c</li>, <li class=\"abc def\">abc</li>]\n",
      "c\n",
      "abc\n"
     ]
    }
   ],
   "source": [
    "#css select  get_text  attrs['id'] ul['id']\n",
    "print(soup.select('.abc'))\n",
    "for li in soup.select('.abc'):\n",
    "    print(li.get_text())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}