学习笔记/xpath解析.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# xpath解析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 最常用且最便捷高效的解析方式\n",
    "# 通用性最强\n",
    "\n",
    "# xpath解析原理\n",
    "# 实例化一个etree的对象，并且需要将被解析的页面源码数据加载到该对象中\n",
    "# 调用etree对象中的xpath方法结合着xpath表达式实现标签定位和内容的捕获\n",
    "\n",
    "# 如何实例化etree对象\n",
    "# etree.parse(filePath) 从本机文件实例化\n",
    "# etree.HTML('page_text') 从互联网获得的html实例化\n",
    "\n",
    "# xpath('xpath表达式')\n",
    "# / 斜杠表示从根节点开始定位 \n",
    "# / 一个斜杠表示一个层级\n",
    "# // 表示多个层级 可以表示从任意位置开始定位\n",
    "\n",
    "# 属性定位 tag[@attrName=\"attrValue\"]\n",
    "\n",
    "# 索引定位 tag[@attrName=\"attrValue\"]/nextAttrName[index] \n",
    "# 注意索引是从 1 开始的\n",
    "\n",
    "# 取文本\n",
    "# /text()  获取标签中直系的文本内容\n",
    "# //text() 获取标签中非直系的文本内容 （所有的文本内容）\n",
    "\n",
    "# 取标签\n",
    "# /@attrName      img/@src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lxml import etree\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.shicimingju.com/'\n",
    "#url = 'https://www.shicimingju.com/chaxun/list/22158.html'\n",
    "page_text = requests.get(url=url, headers=headers).text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/chaxun/zuozhe/1.html', '/chaxun/zuozhe/9.html', '/chaxun/zuozhe/13046.html', '/chaxun/zuozhe/91.html', '/chaxun/zuozhe/77.html', '/chaxun/zuozhe/10.html', '/chaxun/zuozhe/44.html', '/chaxun/zuozhe/43.html', '/chaxun/zuozhe/8.html', '/chaxun/zuozhe/6.html', '/chaxun/zuozhe/11.html', '/chaxun/zuozhe/29.html', '/chaxun/zuozhe/46.html', '/chaxun/zuozhe/81.html', '/chaxun/zuozhe/45.html', '/chaxun/zuozhe/22.html', '/chaxun/zuozhe/113.html', '/chaxun/zuozhe/39.html', '/chaxun/zuozhe/67.html', '/chaxun/zuozhe/15.html', '/chaxun/zuozhe/30.html', '/chaxun/zuozhe/115.html', '/chaxun/zuozhe/42.html', '/chaxun/zuozhe/26.html', '/chaxun/zuozhe/38.html', '/chaxun/zuozhe/666.html', '/chaxun/zuozhe/20.html', '/chaxun/zuozhe/13.html', '/chaxun/zuozhe/66.html', '/chaxun/zuozhe/27.html', '/chaxun/zuozhe/56.html', '/chaxun/zuozhe/129.html', '/chaxun/zuozhe/25.html', '/chaxun/zuozhe/54.html', '/chaxun/zuozhe/5.html', '/chaxun/zuozhe/4623.html', '/chaxun/zuozhe/3.html', '/chaxun/zuozhe/117.html', '/chaxun/zuozhe/99.html', '/chaxun/zuozhe/50.html', '/chaxun/zuozhe/28.html', '/chaxun/zuozhe/223.html', '/chaxun/zuozhe/53.html', '/chaxun/zuozhe/122.html', '/chaxun/zuozhe/32.html', '/chaxun/zuozhe/37.html', '/chaxun/zuozhe/530.html', '/chaxun/zuozhe/4.html', '/chaxun/zuozhe/96.html', '/chaxun/zuozhe/69.html', '/chaxun/zuozhe/60.html', '/chaxun/zuozhe/24.html', '/chaxun/zuozhe/744.html', '/chaxun/zuozhe/51.html', '/chaxun/zuozhe/251.html', '/chaxun/zuozhe/125.html', '/chaxun/zuozhe/102.html', '/chaxun/zuozhe/4676.html', '/chaxun/zuozhe/146.html', '/chaxun/zuozhe/136.html']\n"
     ]
    }
   ],
   "source": [
    "# 实例化一个etree对象 且将这个被解析的源码加载到该对象中\n",
    "tree = etree.HTML(page_text)\n",
    "#r = tree.xpath('/html/body/div')# 返回的是一个Element对象列表\n",
    "#r = tree.xpath('//div')# 寻找源码中所有的div标签\n",
    "\n",
    "#r = tree.xpath('//div[@class=\"shici_content\"]')\n",
    "# r = tree.xpath('//div[@style=\"display: none\"]')\n",
    "\n",
    "#r = tree.xpath('//div[@class=\"card hc_other_zuozhe\"]//li[3]/a/text()')[0]# 获取标签中存储的文本内容\n",
    "#r = tree.xpath('//div[@class=\"card hc_other_zuozhe\"]//text()')\n",
    "r = tree.xpath('//div[@class=\"card hc_other_zuozhe\"]/ul/li/a[1]/@href')\n",
    "print(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# xpath案例\n",
    "import requests\n",
    "from lxml import etree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 爬取到页面源码数据\n",
    "url = 'https://tj.58.com/ershoufang/'\n",
    "headers = {\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'\n",
    "}\n",
    "page_text = requests.get(url=url, headers=headers).text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['161可售富秀园两室，带阁楼，送露台，后期准电梯房'] asdasd\n",
      "['富力又一城，地铁2公里，投资落户  ，津南实验，国展东侧'] asdasd\n",
      "['中海 南大附中 89平三室现房72万起，15天下房本'] asdasd\n",
      "['雍馨花园三居室好楼层随时看房杨村第五   紧挨着文化公园急卖'] asdasd\n",
      "['急售和骏新家园随时看房'] asdasd\n",
      "['换房急售 柳苑里1楼双阳大2室 带小院 出房率高 价格可议'] asdasd\n",
      "['必买好房！八里台一中旁 叠拼上下两层赠送30平米小院  加推'] asdasd\n",
      "['大华国展 地铁旁一号线 南北通透两居室 师大二附小'] asdasd\n",
      "['XIN爱情缤纷里74平2室 低140万 今次一套 随时看房'] asdasd\n",
      "['生态城外国语   雅境园精装修三室车位学籍都有无尾款随时看房'] asdasd\n",
      "['双卫设计 员工 纯洋房2商业2地铁 已成熟小学幼儿园'] asdasd\n",
      "['180万 满五 3室2厅 南北通透 电梯房 融创中央学府'] asdasd\n",
      "['诚心出售 配套齐全 周边完善 先到先得河北区诗景颂苑2室2厅'] asdasd\n",
      "['生态城核心区 铭郡园 滨海小外 精装两室赠车位 业主诚售'] asdasd\n",
      "['新增，瑞江菊苑，两居必看，力荐3楼，震撼价，要买快来看！'] asdasd\n",
      "['新华联世家94平装修三居室110万好楼层可落户。'] asdasd\n",
      "['武清南湖片区  带精装修 随时看房 城区教育 生活配套完善'] asdasd\n",
      "['3室2厅2卫 富力新城 南北通透 129平 居家自住 保养好'] asdasd\n",
      "['水岸公馆 天浦园 满五  改善陪读 好房   价位可谈'] asdasd\n",
      "['3室2厅 鲲贝园 182万 满二 电梯房 133平 房东急置'] asdasd\n",
      "['急急急！世嘉花苑精装59万 送阁楼送露台 老证随时看'] asdasd\n",
      "['北辰区红盘 双地铁 重点学校 宜兴阜   环内新房 央企开发'] asdasd\n",
      "['满二  住房 2室2厅 精装修 电梯房 楼层好 视野无遮挡'] asdasd\n",
      "['业主急售 仁恒滨河湾 通透H两室户型 南开中小学学区 随时看'] asdasd\n",
      "['   西站 地铁1号线 1  钟南开熙悦汇 成熟社区'] asdasd\n",
      "['响螺湾自贸区合景领峰 小高精装交房三室两卫 茱莉亚师大附校'] asdasd\n",
      "['万科东第 新小区 配套齐 高端社区 随时看房'] asdasd\n",
      "['融创别墅 二层赠送挑空面积 带地下室 天津小外学校 随时看房'] asdasd\n",
      "['急售 捡漏 大产权 带房本 精装修 赠车库 翔宇学校学 区房'] asdasd\n",
      "['新房可落户 loft 机场对面 近地铁二号线 即买即住'] asdasd\n",
      "['碧桂园映月庭院 精装两室 家电家具送 楼层好无遮挡'] asdasd\n",
      "['电梯房 南北通透 91平 85万   住房 诚心出售 价格可'] asdasd\n",
      "['西青高新区新房 距离地铁200米 84平三居室 离高铁近'] asdasd\n",
      "['香江棕榈园，两室精装修，中间楼层，赠送阳台面积，近学校'] asdasd\n",
      "['泰丰公园旁桐景园洋房低楼层房主诚心卖看房有钥匙'] asdasd\n",
      "['（低总价）阳光城文澜府,海教园区小户型现世,方正地块,,南开'] asdasd\n",
      "['（必看好房）单价 低的房子 价格可以商量 离学校近'] asdasd\n",
      "['万科洋房，送车位，三个园里总价Zui低，没有对比房源'] asdasd\n",
      "['市政公园旁紧邻地铁 新小区通透三室 电梯洋房 25万装修'] asdasd\n",
      "['出国急售 龙顺园精装修一室8楼 送全部家具家电 有钥匙'] asdasd\n",
      "['蓝水湾两室精装75万拎包入住精装。'] asdasd\n",
      "['海教园 锦绣大家 通地库 纯洋房社区 九年一贯制南开学校'] asdasd\n",
      "['状元之家 宝龙城南苑 通透三室 有学籍 上北师大 配套成熟！'] asdasd\n",
      "['振华里4楼带阁楼，98平中装无税，98万议，看房方便'] asdasd\n",
      "['南开中营小学铁片 精装修 鼓楼商圈 临近海河 明年可上学'] asdasd\n",
      "['98平 满五  住房 首创城星景苑 2室2厅 南北通透 金角'] asdasd\n",
      "['北宁湾宁宇家园(  楼) 75万复试两层100平米可贷款'] asdasd\n",
      "['精装修 电梯房 满二 3室2厅 南北通透 津南新城合安园 房'] asdasd\n",
      "['落户好房，北辰大张庄板块，领雅苑，2室，好楼层，诚心出售'] asdasd\n",
      "['星河领御现房H1户型，落户咨询，上重点学校'] asdasd\n",
      "['房主着急出售，河西务郁  苑 精装修地暖，送全部家具家电'] asdasd\n",
      "['本科 大专 三中学片 天津落户房 两室一厅 好租'] asdasd\n",
      "['静德花园观景三居室 南北通透落地窗全天采光 精装修自住保持好'] asdasd\n",
      "['东疆小外海景房 88.5精装两室 62万 7000一平'] asdasd\n",
      "['95万 南北通透 电梯房 2室2厅 鑫旺里 金角 明厨明卫'] asdasd\n",
      "['生态城鲲贝园精装通透两室134万 随时看房拎包入住'] asdasd\n",
      "['碧桂园 精装修 拎包入住 两居可改三居 可贷款 可落户'] asdasd\n",
      "['沈阳道市场 锦江之星  三星里小区新 双阳通透 户型好'] asdasd\n",
      "['特惠好房！ 南北通透 精装修 3室2厅 诚心出售 价格可议'] asdasd\n",
      "['大学城新房 天津小外学片 小三居 南北通透 适合刚需落户'] asdasd\n",
      "['天津优选落户房大产权私产近学.区落户上学养老均可'] asdasd\n",
      "['港西新城！海河英才落户房！大产权！随时随迁孩 子！有钥匙！'] asdasd\n",
      "['N0.2 落户天津环城四区 北大附属中方學 70年大产权'] asdasd\n",
      "['精装花园洋房 包落户  业主急售 送车位 小院60平 手慢无'] asdasd\n",
      "['29日浦口西里闽侯路xiao学私产满五南北通透4楼精装'] asdasd\n",
      "['靖泰里 南北通透一室  中间楼层  看房随时方便 一梯两户'] asdasd\n",
      "['盛世嘉园 2室1厅1卫 客厅出台 主卧出台'] asdasd\n",
      "['低价急出售，颐安西区低楼层，5中8小，送地下室，可以随时签'] asdasd\n",
      "['柏丽花园 1室1厅1卫 卫生间全明 婚房适宜'] asdasd\n",
      "['光明中学~亨通东区南里 1楼~ 精装通透~送地下室~可落户'] asdasd\n",
      "['捡漏懂不 龙湖紫宸现房 交房所剩不多高品质洋房'] asdasd\n",
      "['捡漏的来！博雅时尚！11楼到顶小高层！两室户型修紧邻卢浮广场'] asdasd\n",
      "['精装 拎包入住 可落户 区政府 配套全 详询   一套'] asdasd\n",
      "['南开中营瑞丽小学 南开中片 公产 总价低有钥匙'] asdasd\n",
      "['南开中心小学片 南开北片 一楼一室 可落戸 看房方便'] asdasd\n",
      "['津九轩  精装修   楼层 明厨明厕 实验一小近地铁800米'] asdasd\n",
      "['央企，保利香颂湖电梯洋房，一楼带小院，广贤路中小学，小院封好'] asdasd\n",
      "['急售~城际旁翠亨南里 ~8小12中学片~送地下室~可落户天津'] asdasd\n",
      "['急售金芳园两室边户 采光好低密度 有钥匙随时看房   户型'] asdasd\n",
      "['海教园区，地铁洋房，上南开中小学九年制学校，永旺商圈。可落户'] asdasd\n",
      "['恒大绿洲花园 正9楼 有小学初中幼儿园 大三室'] asdasd\n",
      "['四小东侧 聚圣里浇筑房 二楼两室 楼下有小房 老证 看房方便'] asdasd\n",
      "['米兰 精装修 可以返租 诚意出售 懂行的给我打电话 带你看房'] asdasd\n",
      "['江南风格观湖景电梯洋房 清盘特惠 现可落户 一楼送院送车位'] asdasd\n",
      "['     远洋城一期 紧邻菜市场 两居室 精装自住 单价低'] asdasd\n",
      "['93万 春风海上 精装修 79平 采光好 周边商圈成熟'] asdasd\n",
      "['豪宅社区中海天空之镜赠送精装修央企实力开发'] asdasd\n",
      "['月牙河地铁旁 义江里 低楼层 改善婚房均可 厅大好用 可贷款'] asdasd\n",
      "['金海园1室73平米100万可议精装老证一楼'] asdasd\n",
      "['21年12月老证 出租中 全款优先'] asdasd\n",
      "['万科特惠房   五套二居三居   综合钜惠6万  精装修'] asdasd\n",
      "['样本间出售，可看水库全景南北通透两居室'] asdasd\n",
      "['国展板块 大华世家 距近地铁1号线  师大二附小   '] asdasd\n",
      "['（现房）首付30万买超大三居室  纯洋房  即买记住随时落户'] asdasd\n",
      "['贻成泰和新都 2室1厅 95万 85平 居家自住'] asdasd\n",
      "['125平 255万 南北通透 中南君悦府 3室2厅 看房方便'] asdasd\n",
      "['南开小学 4室3厅3卫 133.0平米 380.00万元'] asdasd\n",
      "['万科员工房230万起  只限本周  送装修  优惠两个点'] asdasd\n",
      "['降价急售  时运家园 南北通透H户型 次顶楼 采光  好'] asdasd\n",
      "['北辰未来城 新中环内 通透两室双地铁3.5号 婚房改善'] asdasd\n",
      "['鲁能泰山二期  130平  270万 送车位'] asdasd\n",
      "['万达商圈 359万（一梯一户）河东平墅 准现房'] asdasd\n",
      "['时运家园 降价急售 100万'] asdasd\n",
      "['集贤里 银角两室不到顶 南北通透户型 66万 53平房东卖急'] asdasd\n",
      "['红磡领世郡边 联排别墅 豪华装修，送院子送阁楼 诚意出售'] asdasd\n",
      "['     中，。心  南北通透 3室1厅  保康里 置换急售'] asdasd\n",
      "['（落户   无资质可买）京津新城 三条高铁线 44平36万'] asdasd\n",
      "['临湖别墅  送大院 送车位 精装修 业主急售 价格可谈'] asdasd\n",
      "['实房实图 十小五中学片 低楼层 双阳两室 无个税可落户'] asdasd\n",
      "['南开奥体中心 水上公园 双地铁 落户南开 随时看房'] asdasd\n",
      "['团泊现房别墅 使用面积360平 售190万 带车位'] asdasd\n",
      "['住宅 住宅用地 河西户口 精装修 现房 即买即住 租金高！'] asdasd\n",
      "['工农村 3室2厅1卫 114.0平米 170.00万元'] asdasd\n",
      "['落户  电梯房样本间急售 精装修 50万 69平 居家自住'] asdasd\n",
      "['新旭园 户型方正 大两室 前后无遮挡 楼栋位置优势'] asdasd\n",
      "['降价了降价了 海馨园 二期 电梯洋房'] asdasd\n",
      "['房东急售 降价100W 五楼次顶 H户型 满五无差价有钥匙'] asdasd\n",
      "['zhen房源 晓镇家园 两室通透 不挨告诉 低楼层'] asdasd\n",
      "['武清杨村一中板块特价四居泽信公馆十中十二小学片房临城际'] asdasd\n",
      "['来喽 来喽 急卖房源 必看好房 低价格 源 实拍图随时看'] asdasd\n",
      "120\n"
     ]
    }
   ],
   "source": [
    "# 数据解析\n",
    "tree = etree.HTML(page_text)\n",
    "all_house = tree.xpath('//div[@class=\"property\"]')\n",
    "#print(len(all_house))\n",
    "i = 0\n",
    "for house in all_house:\n",
    "    title = house.xpath('.//div[@class=\"property-content-title\"][1]/h3/text()')# . 点表示xpath参照物不再是整个html源码 而是house\n",
    "    print(title, \"asdasd\")\n",
    "    i += 1\n",
    "print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 解析下载图片数据\n",
    "# https://pic.netbian.com/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#<img src=\"/uploads/allimg/210308/224421-1615214661a839.jpg\" alt=\"熊猫 国宝 四川 熊猫4k壁纸\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ISO-8859-1\n",
      "<!doctype html>\r\n",
      "<html>\r\n",
      "<head>\r\n",
      "<meta charset=\"gbk\" />\r\n",
      "<title>4K动物壁纸_高清4K动物图片_彼岸图网</title>\r\n",
      "<meta name=\"keywords\" content=\"4k动物壁纸,高清4k动物壁纸,4k动物图片,4k动物电脑壁纸,4k动物桌面壁纸\" />\r\n",
      "<meta name=\"description\" content=\"彼岸图网免费提供好看的4k动物图片壁纸下载,把高清的4k动物图片壁纸推荐给您,让您更快的找到您想要的4k动物壁纸图片\" />\r\n",
      "<link href=\"/static/css/style.css?2021031112\" rel=\"stylesheet\" type=\"text/css\" />\r\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge,chrome=1\">\r\n",
      "<script type=\"text/javascript\" src=\"/static/js/jquery.min.js\"></script>\r\n",
      "<script type=\"text/javascript\" src=\"/static/js/common.js?2021031112\"></script>\r\n",
      "</head>\r\n",
      "<body>\r\n",
      "<div class=\"header\">\r\n",
      "\t<div class=\"head\">\r\n",
      "\t\t<a href=\"/\" class=\"logo\" title=\"4k壁纸\">首页</a>\r\n",
      "\t\t<ul class=\"nav\">\r\n",
      "<li><a href=\"/new/\" class=\"nav-link\" rel=\"nofollow\">最新</a></li>\r\n",
      "  <li> <a href=\"javascript:;\" class=\"nav-link\">分类</a>\r\n",
      "    <div class=\"nav-m clearfix tran\"><a href=\"/4kfengjing/\" title=\"4K风景图片\">4K风景</a><a href=\"/4kmeinv/\" title=\"4K美女图片\">4K美女</a><a href=\"/4kyouxi/\" title=\"4K游戏图片\">4K游戏</a><a href=\"/4kdongman/\" title=\"4K动漫图片\">4K动漫</a><a href=\"/4kyingshi/\" title=\"4K影视图片\">4K影视</a><a href=\"/4kmingxing/\" title=\"4K明星图片\">4K明星</a><a href=\"/4kqiche/\" title=\"4K汽车图片\">4K汽车</a><a href=\"/4kdongwu/\" class=\"curr\" title=\"4K动物图片\">4K动物</a><a href=\"/4krenwu/\" title=\"4K人物图片\">4K人物</a><a href=\"/4kmeishi/\" title=\"4K美食图片\">4K美食</a><a href=\"/4kzongjiao/\" title=\"4K宗教图片\">4K宗教</a><a href=\"/4kbeijing/\" title=\"4K背景图片\">4K背景</a></div>\r\n",
      "  </li>\r\n",
      "  <li><a href=\"javascript:;\" class=\"nav-link\">尺寸</a><div class=\"nav-m clearfix tran\"> <a href=\"//pic.netbian.com/e/search/result/?searchid=1224\">4K壁纸</a><a href=\"//pic.netbian.com/e/search/result/?searchid=1224\">3840x2160壁纸</a><a href=\"//pic.netbian.com/e/search/result/?searchid=2446\">带鱼屏壁纸</a><a href=\"//pic.netbian.com/e/search/result/?searchid=2446\">3440x1440壁纸</a><a href=\"//pic.netbian.com/e/search/result/?searchid=2453\">5K壁纸</a><a href=\"//pic.netbian.com/e/search/result/?searchid=2453\">5120x2880壁纸</a><a href=\"//pic.netbian.com/e/search/result/?searchid=2543\">8K壁纸</a><a href=\"//pic.netbian.com/e/search/result/?searchid=2543\">7680x4320壁纸</a><a href=\"/shoujibizhi/\">手机壁纸</a><a href=\"//pic.netbian.com/e/search/result/?searchid=12689\" target=\"_blank\">5120x1440壁纸</a></div></li>\r\n",
      " \r\n",
      "  <li><a href=\"/4kdongman/\" class=\"nav-link pc-click\">4K动漫</a></li>\r\n",
      "  <li><a href=\"/4kfengjing/\" class=\"nav-link\" rel=\"nofollow\">4K风景</a></li>\r\n",
      "  <li><a href=\"/vip.html\" class=\"nav-link\" rel=\"nofollow\" style=\"color:#FFA800; target=_blank\">1元会员</a></li>\r\n",
      "</ul>\r\n",
      "<div class=\"search\">\r\n",
      "    <form action=\"/e/search/index.php\" method=\"post\" name=\"schform\" id=\"schform\">\r\n",
      "      <p><input type=\"text\" name=\"keyboard\" value=\"请输入壁纸名称\" onFocus=\"if(this.value=='请输入壁纸名称') this.value='';\" onBlur=\"if(this.value=='') this.value='请输入壁纸名称';\" autocomplete=\"off\"/></p>\r\n",
      "      <input name=\"submit\" type=\"submit\" value=\"\" class=\"sub\" /><input name=\"tempid\" type=\"hidden\" value=\"1\" /><input name=\"tbname\" type=\"hidden\" value=\"photo\" /><input name=\"show\" type=\"hidden\" value=\"title\" />\r\n",
      "    </form>\r\n",
      "   </div>\r\n",
      "<div class=\"loginbar\"><script>document.write('<script src=\"/e/member/login/loginjs.php?t='+Math.random()+'\"><'+'/script>');</script></div>\r\n",
      "</div>\r\n",
      "</div>\r\n",
      "<script>\r\n",
      "var _hmt = _hmt || [];\r\n",
      "(function() {\r\n",
      "  var hm = document.createElement(\"script\");\r\n",
      "  hm.src = \"//hm.baidu.com/hm.js?526caf4e20c21f06a4e9209712d6a20e\";\r\n",
      "  var s = document.getElementsByTagName(\"script\")[0]; \r\n",
      "  s.parentNode.insertBefore(hm, s);\r\n",
      "})();\r\n",
      "</script>\r\n",
      "<div class=\"wrap clearfix\">\r\n",
      "<div id=\"main\">\r\n",
      "<div class=\"loaction\"><span>当前位置：<a href=\"/\">首页</a> > <h1 style=\"display: inline;font-weight: normal;\">4K动物图片</h1></span></div>\r\n",
      "<div class=\"classify clearfix\"> <a href=\"/4kfengjing/\" title=\"4K风景图片\">4K风景</a><a href=\"/4kmeinv/\" title=\"4K美女图片\">4K美女</a><a href=\"/4kyouxi/\" title=\"4K游戏图片\">4K游戏</a><a href=\"/4kdongman/\" title=\"4K动漫图片\">4K动漫</a><a href=\"/4kyingshi/\" title=\"4K影视图片\">4K影视</a><a href=\"/4kmingxing/\" title=\"4K明星图片\">4K明星</a><a href=\"/4kqiche/\" title=\"4K汽车图片\">4K汽车</a><a href=\"/4kdongwu/\" class=\"curr\" title=\"4K动物图片\">4K动物</a><a href=\"/4krenwu/\" title=\"4K人物图片\">4K人物</a><a href=\"/4kmeishi/\" title=\"4K美食图片\">4K美食</a><a href=\"/4kzongjiao/\" title=\"4K宗教图片\">4K宗教</a><a href=\"/4kbeijing/\" title=\"4K背景图片\">4K背景</a></div>\r\n",
      "\t<div class=\"slist\">\r\n",
      "    \t<ul class=\"clearfix\">\r\n",
      "        <li><a href=\"/tupian/26993.html\" target=\"_blank\"><img src=\"/uploads/allimg/210308/224421-1615214661a839.jpg\" alt=\"熊猫 国宝 四川 熊猫4k壁纸\" /><b>熊猫 国宝 四川 熊猫4k壁</b></a></li><li><a href=\"/tupian/26984.html\" target=\"_blank\"><img src=\"/uploads/allimg/210307/232913-16151309534364.jpg\" alt=\"海洋 深海 莫布拉雷4k壁纸\" /><b>海洋 深海 莫布拉雷4k壁</b></a></li><li><a href=\"/tupian/26979.html\" target=\"_blank\"><img src=\"/uploads/allimg/210307/232017-16151304178002.jpg\" alt=\"微软win10 海龟主题4k壁纸3840x2160\" /><b>微软win10 海龟主题4k壁</b></a></li><li><a href=\"/tupian/26516.html\" target=\"_blank\"><img src=\"/uploads/allimg/201111/233047-1605108647d0b0.jpg\" alt=\"大象 动物 岩桥 天然桥 夜 晚上 黑暗 月亮 自然 4k动物图片\" /><b>大象 动物 岩桥 天然桥 </b></a></li><li><a href=\"/tupian/25779.html\" target=\"_blank\"><img src=\"/uploads/allimg/200424/000144-1587657704fe66.jpg\" alt=\"美丽的凤蝶 蝴蝶4k高清壁纸\" /><b>美丽的凤蝶 蝴蝶4k高清壁</b></a></li><li><a href=\"/tupian/25614.html\" target=\"_blank\"><img src=\"/uploads/allimg/200319/005731-158455065162b6.jpg\" alt=\"微软Windows 10马和向日葵风景4k壁纸3840x2160\" /><b>微软Windows 10马和向日</b></a></li><li><a href=\"/tupian/25597.html\" target=\"_blank\"><img src=\"/uploads/allimg/200313/205109-15841038692fed.jpg\" alt=\"蝴蝶特写4k高清图片\" /><b>蝴蝶特写4k高清图片</b></a></li><li><a href=\"/tupian/24568.html\" target=\"_blank\"><img src=\"/uploads/allimg/190808/093323-1565228003d88b.jpg\" alt=\"两只相亲相爱的马4k壁纸\" /><b>两只相亲相爱的马4k壁纸</b></a></li><li><a href=\"/tupian/24017.html\" target=\"_blank\"><img src=\"/uploads/allimg/190413/221558-15551649585352.jpg\" alt=\"日落 云 天空 鸟类 动物 飞行 候鸟 6k图片\" /><b>日落 云 天空 鸟类 动物</b></a></li><li><a href=\"/tupian/24016.html\" target=\"_blank\"><img src=\"/uploads/allimg/190413/220103-15551640639e8f.jpg\" alt=\"七彩神仙5k图片\" /><b>七彩神仙5k图片</b></a></li><li><a href=\"/tupian/23888.html\" target=\"_blank\"><img src=\"/uploads/allimg/190327/211604-1553692564de5c.jpg\" alt=\"丹顶鹤6k图片\" /><b>丹顶鹤6k图片</b></a></li><li><a href=\"/tupian/23798.html\" target=\"_blank\"><img src=\"/uploads/allimg/190316/202152-1552738912b440.jpg\" alt=\"蓝色海洋 海豚 动物7k图片\" /><b>蓝色海洋 海豚 动物7k图</b></a></li><li><a href=\"/tupian/23568.html\" target=\"_blank\"><img src=\"/uploads/allimg/190217/185819-15504010994339.jpg\" alt=\"绵羊 好奇 农场 动物 农村 牧场 6k图片\" /><b>绵羊 好奇 农场 动物 农</b></a></li><li><a href=\"/tupian/23567.html\" target=\"_blank\"><img src=\"/uploads/allimg/190217/185005-1550400605759e.jpg\" alt=\"两只北极熊 浪漫情侣 吻 耳语 4K图片\" /><b>两只北极熊 浪漫情侣 吻</b></a></li><li><a href=\"/tupian/23563.html\" target=\"_blank\"><img src=\"/uploads/allimg/190217/132235-1550380955df50.jpg\" alt=\"冰岛美丽的马4k壁纸\" /><b>冰岛美丽的马4k壁纸</b></a></li><li><a href=\"/tupian/23550.html\" target=\"_blank\"><img src=\"/uploads/allimg/190213/220959-1550066999b8c7.jpg\" alt=\"日落小男孩狮子梦想4k壁纸\" /><b>日落小男孩狮子梦想4k壁</b></a></li><li><a href=\"/tupian/22965.html\" target=\"_blank\"><img src=\"/uploads/allimg/181129/112650-154346201044ff.jpg\" alt=\"自然 鸟儿 绿色背景4k动物壁纸\" /><b>自然 鸟儿 绿色背景4k动</b></a></li><li><a href=\"/tupian/22042.html\" target=\"_blank\"><img src=\"/uploads/allimg/180905/103725-153611504540ea.jpg\" alt=\"蜂鸟 绿色背景4k壁纸\" /><b>蜂鸟 绿色背景4k壁纸</b></a></li><li><a href=\"/tupian/20694.html\" target=\"_blank\"><img src=\"/uploads/allimg/180531/214948-1527774588db9a.jpg\" alt=\"海边 海水 海滩 三匹马5k图片\" /><b>海边 海水 海滩 三匹马5</b></a></li><li><a href=\"/tupian/20330.html\" target=\"_blank\"><img src=\"/uploads/allimg/180510/220123-15259608835d26.jpg\" alt=\"黑暗中发光 蓝色蝴蝶3440x1440唯美壁纸\" /><b>黑暗中发光 蓝色蝴蝶344</b></a></li>\r\n",
      "    \t</ul>\r\n",
      "    </div>\r\n",
      "\t<div class=\"page\"><b>1</b><a href=\"/4kdongwu/index_2.html\">2</a><a href=\"/4kdongwu/index_3.html\">3</a><a href=\"/4kdongwu/index_4.html\">4</a><a href=\"/4kdongwu/index_5.html\">5</a><a href=\"/4kdongwu/index_6.html\">6</a><a href=\"/4kdongwu/index_7.html\">7</a><span class=\"slh\">…</span><a href=\"/4kdongwu/index_21.html\">21</a><a href=\"/4kdongwu/index_2.html\">下一页</a></div>\r\n",
      "\r\n",
      "<div>\r\n",
      "<a href=\"https://ibaotu.com/guanggao/1-0-0-0-3-1.html?spm=batw1\" target=\"_blank\"><img src=\"/d/file/2021/03/19/2357235DMKe.jpg\" width=\"1200\" height=\"90\" border=\"0\"/></a>\r\n",
      "</div>\r\n",
      "    </div>\r\n",
      "</div>\r\n",
      "<div id=\"footer\">\r\n",
      "<div class=\"w\"><p>彼岸图网为用户免费分享产生,若发现您的权利被侵害,我们尽快处理 客服QQ:55346968 <a target=\"_blank\" href=\"//wpa.qq.com/msgrd?v=3&uin=55346968&site=qq&menu=yes\"><img border=\"0\" src=\"/static/img/qqonline.jpg\" alt=\"点击这里给我发消息\" title=\"点击这里给我发消息\"/></a>（工作时间：9:00-23:00）4K壁纸QQ群：20379343、427389122</p>\r\n",
      "<p>Copyright &copy; 2021 <span class=\"sitename\"><a href=\"//pic.netbian.com/\" title=\"彼岸图网\">彼岸图网</a></span> <a href=\"//www.beian.miit.gov.cn/\" target=\"_blank\">闽ICP备13013111号-1</a> <a href=\"//www.netbian.com/\" target=\"_blank\" title=\"壁纸\">壁纸</a> <a href=\"//pic.netbian.com/\" title=\"4k\">4k</a> <a href=\"//pic.netbian.com/\" title=\"4k壁纸\">4k壁纸</a> <a href=\"//pic.netbian.com/\" target=\"_blank\" title=\"壁纸\">壁纸</a> <a href=\"//pic.netbian.com/\" target=\"_blank\" title=\"高清壁纸\">高清壁纸</a> <a href=\"//pic.netbian.com/\" target=\"_blank\" title=\"电脑壁纸\">电脑壁纸</a> <a href=\"/bangzhu.html\" target=\"_blank\">帮助</a> <a href=\"/tougao.html\" target=\"_blank\">原创投稿</a>\r\n",
      "</p>\r\n",
      "</div>\r\n",
      "</div> \r\n",
      "<div class=\"gotop\">\r\n",
      "<div class=\"bd\">\r\n",
      "<div class=\"btn-qq\"><i></i>客服<div class=\"kf-trop kf-qq\"><p class=\"kf-name\">在线客服</p><p>工作时间(9:00-23:00)</p><p class=\"kf-img\"><a href=\"//wpa.qq.com/msgrd?v=3&uin=55346968&site=qq&menu=yes\"><img border=\"0\" src=\"/static/img/qqonline.jpg\" alt=\"点击这里给我发消息\" title=\"点击这里给我发消息\"/></a></p><p class=\"kf-name kf-faq\"><a href=\"/bangzhu.html\" target=\"_blank\">常见问题>></a></p></div></div>\r\n",
      "<a href=\"javascript:;\" class=\"btn-btn btn-phone\"><i></i>电话<div class=\"kf-trop\"><p>服务热线</p><b>13605998361</b><p>工作时间(9:00-23:00)</p></div></a>\r\n",
      "<div class=\"btn-btn btn-group\">\r\n",
      "                    <i></i>QQ群<div class=\"kf-trop\"><h4>4K壁纸QQ群</h4>\r\n",
      "                    <p><span>壁纸群4（20379343）</span><a>最新</a></p>\r\n",
      "                    <p><span>壁纸群3（223739596）</span><a>最新</a></p>\r\n",
      "                    <p><span>壁纸群2（912383939）</span><a>最火</a></p>\r\n",
      "                    <p><span>壁纸群1（427389122）</span><a>热门</a></p>\r\n",
      "                    <div class=\"more\"><a href=\"\">更多</a></div>\r\n",
      "                    </div>\r\n",
      "                </div>\r\n",
      "<div class=\"btn-btn btn-weixin\"><i></i>全面屏<div class=\"kf-trop\"><img src=\"/static/img/weixinqrcode.jpg\" /><p>全面屏手机壁纸</p></div></div>\r\n",
      "</div>\r\n",
      "<a href=\"javascript:;\" class=\"btn-btn btn-top\"><i></i>顶部</a>\r\n",
      "</div><script>\r\n",
      "var _hmt = _hmt || [];\r\n",
      "(function() {\r\n",
      "  var hm = document.createElement(\"script\");\r\n",
      "  hm.src = \"https://hm.baidu.com/hm.js?526caf4e20c21f06a4e9209712d6a20e\";\r\n",
      "  var s = document.getElementsByTagName(\"script\")[0]; \r\n",
      "  s.parentNode.insertBefore(hm, s);\r\n",
      "})();\r\n",
      "</script>\r\n",
      "<script type=\"text/javascript\" src=\"/static/js/tb.js\"></script>\r\n",
      "</body>\r\n",
      "</html>\r\n",
      "<script src='/e/public/onclick/?enews=doclass&classid=58'></script>\n"
     ]
    }
   ],
   "source": [
    "url = 'https://pic.netbian.com/4kdongwu/'\n",
    "headers = {\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'\n",
    "}\n",
    "\n",
    "response = requests.get(url=url, headers=headers)\n",
    "\n",
    "# 中文乱码问题处理\n",
    "\n",
    "# 手动设定相应数据的编码格式\n",
    "print(response.encoding)\n",
    "response.encoding = 'gbk'\n",
    "\n",
    "page_text = response.text\n",
    "print(page_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "if not os.path.exists('./picLibs'):\n",
    "    os.mkdir('./picLibs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "熊猫 国宝 四川 熊猫4k壁纸.jpg 下载成功。\n",
      "海洋 深海 莫布拉雷4k壁纸.jpg 下载成功。\n",
      "微软win10 海龟主题4k壁纸3840x2160.jpg 下载成功。\n",
      "大象 动物 岩桥 天然桥 夜 晚上 黑暗 月亮 自然 4k动物图片.jpg 下载成功。\n",
      "美丽的凤蝶 蝴蝶4k高清壁纸.jpg 下载成功。\n",
      "微软Windows 10马和向日葵风景4k壁纸3840x2160.jpg 下载成功。\n",
      "蝴蝶特写4k高清图片.jpg 下载成功。\n",
      "两只相亲相爱的马4k壁纸.jpg 下载成功。\n",
      "日落 云 天空 鸟类 动物 飞行 候鸟 6k图片.jpg 下载成功。\n",
      "七彩神仙5k图片.jpg 下载成功。\n",
      "丹顶鹤6k图片.jpg 下载成功。\n",
      "蓝色海洋 海豚 动物7k图片.jpg 下载成功。\n",
      "绵羊 好奇 农场 动物 农村 牧场 6k图片.jpg 下载成功。\n",
      "两只北极熊 浪漫情侣 吻 耳语 4K图片.jpg 下载成功。\n",
      "冰岛美丽的马4k壁纸.jpg 下载成功。\n",
      "日落小男孩狮子梦想4k壁纸.jpg 下载成功。\n",
      "自然 鸟儿 绿色背景4k动物壁纸.jpg 下载成功。\n",
      "蜂鸟 绿色背景4k壁纸.jpg 下载成功。\n",
      "海边 海水 海滩 三匹马5k图片.jpg 下载成功。\n",
      "黑暗中发光 蓝色蝴蝶3440x1440唯美壁纸.jpg 下载成功。\n"
     ]
    }
   ],
   "source": [
    "tree = etree.HTML(page_text)\n",
    "# 数据解析：src的属性值 alt的属性值\n",
    "li_list = tree.xpath('//div[@class=\"slist\"]/ul/li')# 获取所有li标签\n",
    "for li in li_list:\n",
    "    img_src = 'https://pic.netbian.com/' + li.xpath('./a/img/@src')[0]\n",
    "    img_name = li.xpath('./a/img/@alt')[0] + '.jpg'\n",
    "    # 通用处理中文乱码的解决方式\n",
    "    #img_name = img_name.encode('iso-8859-1').decode('gbk')\n",
    "    #print(img_name, img_src)# 乱码问题\n",
    "    # 持久化存储\n",
    "    img_data = requests.get(url=img_src, headers=headers).content\n",
    "    img_path = 'picLibs/' + img_name\n",
    "    with open(img_path, 'wb') as fp:\n",
    "        fp.write(img_data)\n",
    "        print(img_name, '下载成功。')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 城市名称"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.aqistudy.cn/historydata/'\n",
    "headers = {\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'\n",
    "}\n",
    "page_text = requests.get(url=url, headers=headers).text\n",
    "tree = etree.HTML(page_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 热门城市\n",
    "li_list = tree.xpath('//div[@class=\"bottom\"]/ul/li')\n",
    "allCityNames = []\n",
    "for li in li_list:\n",
    "    hot_city_name = li.xpath('./a/text()')[0]# 城市名称\n",
    "    allCityNames.append(hot_city_name)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 所有城市\n",
    "city_names = tree.xpath('//div[@class=\"bottom\"]/ul/div[2]/li')\n",
    "for li in city_names:\n",
    "    city_name = li.xpath('./a/text()')[0]\n",
    "    allCityNames.append(city_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "北京\n",
      "上海\n",
      "广州\n",
      "深圳\n",
      "杭州\n",
      "天津\n",
      "成都\n",
      "南京\n",
      "西安\n",
      "武汉\n",
      "阿坝州\n",
      "安康\n",
      "阿克苏地区\n",
      "阿里地区\n",
      "阿拉善盟\n",
      "阿勒泰地区\n",
      "安庆\n",
      "安顺\n",
      "鞍山\n",
      "克孜勒苏州\n",
      "安阳\n",
      "蚌埠\n",
      "白城\n",
      "保定\n",
      "北海\n",
      "宝鸡\n",
      "北京\n",
      "毕节\n",
      "博州\n",
      "白山\n",
      "百色\n",
      "保山\n",
      "白沙\n",
      "包头\n",
      "保亭\n",
      "本溪\n",
      "巴彦淖尔\n",
      "白银\n",
      "巴中\n",
      "滨州\n",
      "亳州\n",
      "长春\n",
      "昌都\n",
      "常德\n",
      "成都\n",
      "承德\n",
      "赤峰\n",
      "昌吉州\n",
      "五家渠\n",
      "昌江\n",
      "澄迈\n",
      "重庆\n",
      "长沙\n",
      "常熟\n",
      "楚雄州\n",
      "朝阳\n",
      "沧州\n",
      "长治\n",
      "常州\n",
      "潮州\n",
      "郴州\n",
      "池州\n",
      "崇左\n",
      "滁州\n",
      "定安\n",
      "丹东\n",
      "东方\n",
      "东莞\n",
      "德宏州\n",
      "大理州\n",
      "大连\n",
      "大庆\n",
      "大同\n",
      "定西\n",
      "大兴安岭地区\n",
      "德阳\n",
      "东营\n",
      "黔南州\n",
      "达州\n",
      "德州\n",
      "儋州\n",
      "鄂尔多斯\n",
      "恩施州\n",
      "鄂州\n",
      "防城港\n",
      "佛山\n",
      "抚顺\n",
      "阜新\n",
      "阜阳\n",
      "富阳\n",
      "抚州\n",
      "福州\n",
      "广安\n",
      "贵港\n",
      "桂林\n",
      "果洛州\n",
      "甘南州\n",
      "固原\n",
      "广元\n",
      "贵阳\n",
      "甘孜州\n",
      "赣州\n",
      "广州\n",
      "淮安\n",
      "海北州\n",
      "鹤壁\n",
      "淮北\n",
      "河池\n",
      "海东地区\n",
      "邯郸\n",
      "哈尔滨\n",
      "合肥\n",
      "鹤岗\n",
      "黄冈\n",
      "黑河\n",
      "红河州\n",
      "怀化\n",
      "呼和浩特\n",
      "海口\n",
      "呼伦贝尔\n",
      "葫芦岛\n",
      "哈密地区\n",
      "海门\n",
      "海南州\n",
      "淮南\n",
      "黄南州\n",
      "衡水\n",
      "黄山\n",
      "黄石\n",
      "和田地区\n",
      "海西州\n",
      "河源\n",
      "衡阳\n",
      "汉中\n",
      "杭州\n",
      "菏泽\n",
      "贺州\n",
      "湖州\n",
      "惠州\n",
      "吉安\n",
      "金昌\n",
      "晋城\n",
      "景德镇\n",
      "金华\n",
      "西双版纳州\n",
      "九江\n",
      "吉林\n",
      "即墨\n",
      "江门\n",
      "荆门\n",
      "佳木斯\n",
      "济南\n",
      "济宁\n",
      "胶南\n",
      "酒泉\n",
      "句容\n",
      "湘西州\n",
      "金坛\n",
      "鸡西\n",
      "嘉兴\n",
      "江阴\n",
      "揭阳\n",
      "济源\n",
      "嘉峪关\n",
      "胶州\n",
      "焦作\n",
      "锦州\n",
      "晋中\n",
      "荆州\n",
      "库尔勒\n",
      "开封\n",
      "黔东南州\n",
      "克拉玛依\n",
      "昆明\n",
      "喀什地区\n",
      "昆山\n",
      "临安\n",
      "六安\n",
      "来宾\n",
      "聊城\n",
      "临沧\n",
      "娄底\n",
      "乐东\n",
      "廊坊\n",
      "临汾\n",
      "临高\n",
      "漯河\n",
      "丽江\n",
      "吕梁\n",
      "陇南\n",
      "六盘水\n",
      "拉萨\n",
      "乐山\n",
      "丽水\n",
      "凉山州\n",
      "陵水\n",
      "莱芜\n",
      "莱西\n",
      "临夏州\n",
      "溧阳\n",
      "辽阳\n",
      "辽源\n",
      "临沂\n",
      "龙岩\n",
      "洛阳\n",
      "连云港\n",
      "莱州\n",
      "兰州\n",
      "林芝\n",
      "柳州\n",
      "泸州\n",
      "马鞍山\n",
      "牡丹江\n",
      "茂名\n",
      "眉山\n",
      "绵阳\n",
      "梅州\n",
      "宁波\n",
      "南昌\n",
      "南充\n",
      "宁德\n",
      "内江\n",
      "南京\n",
      "怒江州\n",
      "南宁\n",
      "南平\n",
      "那曲地区\n",
      "南通\n",
      "南阳\n",
      "平度\n",
      "平顶山\n",
      "普洱\n",
      "盘锦\n",
      "蓬莱\n",
      "平凉\n",
      "莆田\n",
      "萍乡\n",
      "濮阳\n",
      "攀枝花\n",
      "青岛\n",
      "琼海\n",
      "秦皇岛\n",
      "曲靖\n",
      "齐齐哈尔\n",
      "七台河\n",
      "黔西南州\n",
      "清远\n",
      "庆阳\n",
      "钦州\n",
      "衢州\n",
      "泉州\n",
      "琼中\n",
      "荣成\n",
      "日喀则\n",
      "乳山\n",
      "日照\n",
      "韶关\n",
      "寿光\n",
      "上海\n",
      "绥化\n",
      "石河子\n",
      "石家庄\n",
      "商洛\n",
      "三明\n",
      "三门峡\n",
      "山南\n",
      "遂宁\n",
      "四平\n",
      "商丘\n",
      "宿迁\n",
      "上饶\n",
      "汕头\n",
      "汕尾\n",
      "绍兴\n",
      "三亚\n",
      "邵阳\n",
      "沈阳\n",
      "十堰\n",
      "松原\n",
      "双鸭山\n",
      "深圳\n",
      "朔州\n",
      "宿州\n",
      "随州\n",
      "苏州\n",
      "石嘴山\n",
      "泰安\n",
      "塔城地区\n",
      "太仓\n",
      "铜川\n",
      "屯昌\n",
      "通化\n",
      "天津\n",
      "铁岭\n",
      "通辽\n",
      "铜陵\n",
      "吐鲁番地区\n",
      "铜仁地区\n",
      "唐山\n",
      "天水\n",
      "太原\n",
      "台州\n",
      "泰州\n",
      "文昌\n",
      "文登\n",
      "潍坊\n",
      "瓦房店\n",
      "威海\n",
      "乌海\n",
      "芜湖\n",
      "武汉\n",
      "吴江\n",
      "乌兰察布\n",
      "乌鲁木齐\n",
      "渭南\n",
      "万宁\n",
      "文山州\n",
      "武威\n",
      "无锡\n",
      "温州\n",
      "吴忠\n",
      "梧州\n",
      "五指山\n",
      "西安\n",
      "兴安盟\n",
      "许昌\n",
      "宣城\n",
      "襄阳\n",
      "孝感\n",
      "迪庆州\n",
      "锡林郭勒盟\n",
      "厦门\n",
      "西宁\n",
      "咸宁\n",
      "湘潭\n",
      "邢台\n",
      "新乡\n",
      "咸阳\n",
      "新余\n",
      "信阳\n",
      "忻州\n",
      "徐州\n",
      "雅安\n",
      "延安\n",
      "延边州\n",
      "宜宾\n",
      "盐城\n",
      "宜昌\n",
      "宜春\n",
      "银川\n",
      "运城\n",
      "伊春\n",
      "云浮\n",
      "阳江\n",
      "营口\n",
      "榆林\n",
      "玉林\n",
      "伊犁哈萨克州\n",
      "阳泉\n",
      "玉树州\n",
      "烟台\n",
      "鹰潭\n",
      "义乌\n",
      "宜兴\n",
      "玉溪\n",
      "益阳\n",
      "岳阳\n",
      "扬州\n",
      "永州\n",
      "淄博\n",
      "自贡\n",
      "珠海\n",
      "湛江\n",
      "镇江\n",
      "诸暨\n",
      "张家港\n",
      "张家界\n",
      "张家口\n",
      "周口\n",
      "驻马店\n",
      "章丘\n",
      "肇庆\n",
      "中山\n",
      "舟山\n",
      "昭通\n",
      "中卫\n",
      "张掖\n",
      "招远\n",
      "资阳\n",
      "遵义\n",
      "枣庄\n",
      "漳州\n",
      "郑州\n",
      "株洲\n"
     ]
    }
   ],
   "source": [
    "for name in allCityNames:\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 一次性获取所有城市名称"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.aqistudy.cn/historydata/'\n",
    "headers = {\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'\n",
    "}\n",
    "page_text = requests.get(url=url, headers=headers).text\n",
    "tree = etree.HTML(page_text)\n",
    "allCityNames = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 解析到热门城市和所有城市对应的a标签\n",
    "# 层级关系\n",
    "# div/ul/li/a\n",
    "# div/ul/div[2]/li/a\n",
    "a_list = tree.xpath('//dic[@class=\"bottom\"]/ul/li/a | //div[@class=\"bottom\"]/ul/div[2]/li')# 使用 | 使得两个xpath表达式可以同时作用\n",
    "for a in a_list:\n",
    "    name = a.xpath('./text()')[0]\n",
    "    allCityNames.append(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "384\n"
     ]
    }
   ],
   "source": [
    "print(len(allCityNames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# xpath 的作业\n",
    "# 结束"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}