async 异步抓取 花瓣网高清大图 30s爬取500张

废话 不多说，直接上代码，不懂得看注释

先安装  pip install aiohttp

"异步抓取花瓣网图片"

# pip install aiohttp
import requests
import aiohttp
import asyncio
import time
import os

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36',
    "X-Request": "JSON",
    "Accept": "application/json",
    "X-Requested-With": "XMLHttpRequest"
}


def get_image_urls():
    """获取图片链接"""
    print('开始获取图片链接，请耐心等待......')
    image_id = 2551285279
    for page in range(1,26):
        url = "https://huaban.com/boards/41743806/?jzwfs8ej&max="+ str(image_id) +"&limit=20&wfl=1"
        response = requests.get(url,headers=headers).json()
        pins = response['board']['pins']
        end_pins_id = pins[-1]['pin_id']
        for i in pins:
            key = i['file']['key']
            urls.append('http://hbimg.huabanimg.com/'+key)
        image_id = end_pins_id  # 下一页url中需要的参数


#特殊的函數：该函数调用后,函数内部的程序语句不会被执行,但是该函数调用会返回一个协程对象
async def get_audio_data(url):
  #使用aiohttp进行请求发送
    #实例化了一个发送网络请求的对象
    async with aiohttp.ClientSession() as s:
      #该函数内部的异步操作必须使用await进行修饰
        async with await s.get(url=url,headers=headers) as response:
            audio_data = await response.read()  #read()返回的是二进制形式的响应数据
            return {'data':audio_data,'url':url}



#任务对象的回调函数,进行数据的持久化存储
def saveData(task):
    dic_obj = task.result()
    name = dic_obj['url'].split('/')[-1]
    data = dic_obj['data']
    image_dir = 'images'
    if not os.path.exists(image_dir):
        os.mkdir(image_dir)
    with open(os.path.join(image_dir,name) + '.jpg','wb') as fp:
        fp.write(data)
    print(name+'下载完毕!')


if __name__ == '__main__':
    start_time = time.clock()
    urls = []
    tasks = []
    get_image_urls()
    for url in urls:
        #调用该特殊函数,让其返回一个协程对象
        c = get_audio_data(url)
        #将协程对象封装到任务对象中
        task = asyncio.ensure_future(c)
        # 给任务对象绑定回调函数
        task.add_done_callback(saveData)
        #将任务对象添加到列表中
        tasks.append(task)
    #创建一个事件循环对象
    loop = asyncio.get_event_loop()
    #将任务对象列表注册到事件循环对象中,并且开启事件循环
    loop.run_until_complete(asyncio.wait(tasks))
    end_time = time.clock()
    print('抓取{}张图片，共计用时{}秒'.format(len(tasks),end_time-start_time))

注：window最大线程数 512，所以任务数不要超过这个值，否则 抛出异常