基于线程池的梨视频爬取

需求：爬取梨视频中的视频数据

https://www.pearvideo.com/category_4

代码如下

import requests
from lxml import etree
import re
import random
%%time
from multiprocessing.dummy import Pool #线程池
#实例化一个线程池对象
pool = Pool(10)#参数表示的是开启线程的个数
#爬取梨视频的视频数据
url='https://www.pearvideo.com/category_4'
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
}
page_text=requests.get(url=url,headers=headers).text
#解析 详情url 视频的标题
tree = etree.HTML(page_text)  
#定位视屏详情的标签元素
li_list=tree.xpath('//ul[@id="listvideoListUl"]/li')
video_url_list=[]
for li in li_list:
    detail_url='https://www.pearvideo.com/'+ li.xpath('./div/a/@href')[0]
    detail_text=requests.get(url=detail_url,headers=headers).text
     #解析：视屏的url
     # srcUrl在哪？ 找寻路径，点击详情页面开发者模式，network--doc---video--mp4
    exp='srcUrl="(.*?)",'
    video_url=re.findall(exp,detail_text,re.S)[0]
    video_url_list.append(video_url)
#使用线程池进行视频数据并发下载
video_data_list=pool.map(lambda link:requests.get(url=link,headers=headers).content,video_url_list)

pool.map(saveData,video_data_list)
pool.close()
pool.join()
    

def saveData(data):
    name = str(random.randint(1,10000))+'.mp4'
    with open(name,'wb') as fp:
        fp.write(data)
        print(name+'下载成功')