爬虫--XPATH解析

今天说一下关于爬取数据解析的方式---->XPATH,XPATH是解析方式中最重要的一种方式

1.安装：pip install lxml

 2.原理

　　1. 获取页面源码数据

　　2.实例化一个etree的对象，并且将页面源码数据加载到该对象中

　　3.调用该对象的xpath方法进行指定标签的定位

　　4.注意：xpath函数必须结合着xpath表达式进行标签定位和内容捕获

说了也不明白，直接上例子！！！！

1.解析58二手房的相关数据

#引用requests
import requests
#引用lxml
from lxml import etree
#地址
url = 'https://bj.58.com/ershoufang/sub/l16/s2242/?utm_source=market&spm=u-2d2yxv86y3v43nkddh1.bdpcpz_bt&PGTID=0d30000c-0000-1139-b00c-643d0d315a04&ClickID=1'
#伪装的请求头，证明我是浏览器
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
#获取当前整个页面
page_text = requests.get(url,headers=headers).text
#作用于当前页面之后再解析数据
tree = etree.HTML(page_text)
#//ul[@class="house-list-wrap"]/li这就是lxml解析，//代表前面的层次
li_list = tree.xpath('//ul[@class="house-list-wrap"]/li')
# print(li_list)#得到每一个<Element li at 0x202a8c62288>这玩意
#再次循环
for li in li_list:
#再次解析得到准确的数据！！！
    title = li.xpath('./div[2]/h2[1]/a/text()')[0]
    print(title)

2.福利福利！！！！下载彼岸图网中的图片数据

import os
import requests
from lxml import etree
#这里注意，这是python3中的写法！！！
import urllib.request
url = 'http://pic.netbian.com/4kmeinv/'
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
#二话不说直接发情再说
response = requests.get(url,headers=headers)

#如果当前根目录下没有imgs文件夹，就创建！！！
if not os.path.exists('./imgs'):
    os.mkdir('./imgs')
    
#得到请求数据
page_text = response.text
#作用当前页面
tree = etree.HTML(page_text)
#lxml解析
li_list = tree.xpath('//div[@class="slist"]/ul/li')
#循环得到准确的数据
for li in li_list:
    img_name = li.xpath('./a/b/text()')[0]
    # 处理中文乱码！不要理解记住就ok
    img_name = img_name.encode('ISO-8859-1').decode('gbk')
    #拼接完整的地址
    img_url = 'http://pic.netbian.com' + li.xpath('./a/img/@src')[0]
    #图片的名字
    img_path = './imgs/' + img_name + '.jpg'
    #这里避免打开文件就用urllib直接写入
    urllib.request.urlretrieve(url=img_url,filename=img_path)

3.解析所有城市名称（https://www.aqistudy.cn/historydata/）

import requests
from lxml import etree

url = 'https://www.aqistudy.cn/historydata/'

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}

res = requests.get(url=url, headers=headers).text
tree = etree.HTML(res)
city_list = tree.xpath('//div[@class="bottom"]/ul/li/a/text() | //div[@class="bottom"]/ul/div[2]/li/a/text()')        # 逻辑运算符，这里 | 表示或的关系
city = ''.join(city_list)

4.煎蛋网的爬去图片

# 煎蛋网图片
import requests
from lxml import etree
#base对于加魔数据进行解密
import base64
import os
import urllib.request

if not os.path.exists('./jiandan'):
    os.mkdir('./jiandan')

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
url = 'http://jandan.net/ooxx'

res1 = requests.get(url, headers=headers).text
tree = etree.HTML(res1)

span_list = tree.xpath('//span[@class="img-hash"]/text()')
for span_hash in span_list:
    #对于加密数据进行解密，编码是utf-8并且拼接完整的url
    img_url = 'http:' + base64.b64decode(span_hash).decode('utf8')
    #得到具体的数据
    img_data = requests.get(url=img_url, headers=headers).content

    filepath = './jiandan/' + img_url.split('/')[-1]
    urllib.request.urlretrieve(url=img_url, filename=filepath)
    print(filepath, '下载完成！')

print('over')

5.爬去简历模板

import requests
from lxml import etree
import random
import os

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}

if not os.path.exists('./jianli'):
    os.mkdir('./jianli')
#爬去前4页的数据
for i in range(1, 4):
    if i == 1:
        #第一页数据
        url = 'http://sc.chinaz.com/jianli/free.html'
    else:
        #不是第一页的数据，进行数据的拼接
        url = 'http://sc.chinaz.com/jianli/free_%s.html' % (i)

    response = requests.get(url=url, headers=headers)
    #字符编码改一下,否则出现这种问题：求职电子版简历免费下载 下载完成！
    response.encoding = 'utf8'

    res = response.text

    tree = etree.HTML(res)

    a_list = tree.xpath('//a[@class="title_wl"]')
    for a in a_list:
        name = a.xpath('./text()')[0]
        jl_url = a.xpath('./@href')[0]

        response = requests.get(url=jl_url, headers=headers)
        response.encoding = 'utf8'
        res1 = response.text
        tree = etree.HTML(res1)
        download_url_list = tree.xpath('//div[@class="clearfix mt20 downlist"]/ul/li/a/@href')
        download_url = random.choice(download_url_list)

        res3 = requests.get(url=download_url, headers=headers).content

        filepath = './jianli/' + name + '.rar'
        #如果上边是content，写入的时候记得’wb'
        with open(filepath, 'wb') as f:
            f.write(res3)
        print(name, '下载完成！')

print('over')

6.站长直接图片下载，图片懒加载

import requests
from lxml import etree
import os
import urllib
import urllib.request

if not os.path.exists('./tupian'):
    os.mkdir('./tupian')

url = 'http://sc.chinaz.com/tupian/'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}

response = requests.get(url=url, headers=headers)
response.encoding = 'utf8'
res = response.text
tree = etree.HTML(res)
url_list = tree.xpath('//div[@id="container"]/div/div/a/img/@src2')  # img标签是伪属性src2，当图片滚动到视野内时变为 src

for url in url_list:
    filepath = './tupian/' + url.rsplit('/', 1)[-1]
    urllib.request.urlretrieve(url, filepath)
    print(filepath, '下载完成！')

print('over')