爬取梨视频网页的视频

"""
目标:爬取梨视频世界板块的视频并下载到本地
分析:数据为动态加载的
"""
import requests
from lxml import etree
import re
import os

dirName = './videoLibs'
if not os.path.exists(dirName):
    os.mkdir(dirName)

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36'
}
url = 'https://www.pearvideo.com/category_2'
# 获取世界板块页面源码
page_text = requests.get(url, headers=headers).text
# 解析
tree = etree.HTML(page_text)
li_list = tree.xpath('//*[@id="categoryList"]/li')

for li in li_list:
    video_url = 'https://www.pearvideo.com/'+li.xpath('./div/a/@href')[0]
    video_title = li.xpath('./div/a/@href')[0] + '.mp4'    # 标题中多含特殊字符,不能作为文件命名,用视频编号代替一下...
    # 获取视频播放页面源码
    video_page_text = requests.get(video_url, headers=headers).text
    # 发现视频链接存放在js代码中,所以不能使用xpath解析,改用正则!
    ex = 'srcUrl="(.*?)",vdoUrl'
    detail_url = re.findall(ex, video_page_text, re.S)[0]
    # 向视频链接发起get请求下载视频到本地
    video_data = requests.get(detail_url, headers=headers).content
    video_path = dirName + '/' + video_title
    with open(video_path, 'wb') as fp:
        fp.write(video_data)