异步爬虫批量下载图片,文件下载链接已失效,不要直接运行
# 异步批量下载 import aiohttp import asyncio import time async def job(session, url): # 声明为异步函数 name = url.split('/')[-1] # 获得名字 img = await session.get(url) # 触发到await就切换,等待get到数据 imgcode = await img.read() # 读取内容 with open("tmp/"+str(name), 'wb') as f: f.write(imgcode) return str(url) async def main(loop, urls): async with aiohttp.ClientSession() as session: # 建立会话session tasks = [loop.create_task(job(session, url)) for url in urls] # 建立所有任务 finished, unfinished = await asyncio.wait(tasks) # 触发await,等待任务完成 all_results = [r.result() for r in finished] # 获取所有结果 print("ALL RESULT:"+str(all_results)) urls = ['https://pythondict.com/wp-content/uploads/2019/07/2019073115192114.jpg', 'https://pythondict.com/wp-content/uploads/2019/07/2019080216113098.jpg'] loop = asyncio.get_event_loop() loop.run_until_complete(main(loop, urls)) loop.close()
参考资料: