【爬虫】必应图片按关键词进行图片下载

"""
关键是找到正确的url，然后判断组成url的参数的规律，多观察几个url即可得出规律
"""

import requests
from fake_useragent import UserAgent
import os
from lxml import etree
import json
import time

headers = {'User-Agent': UserAgent().random}

keyword = "周星驰"
url = 'https://cn.bing.com/images/async?q={}&first={}&count=35&relp={}&tsc=ImageHoverTitle&mmasync=1'

if not os.path.exists(keyword):
    os.mkdir(keyword)
os.chdir(keyword)

first = 35
relp = 35
count = 1

while True:
    try:
        html_str = requests.get(url.format(keyword, first, relp), headers=headers).content.decode()
        html = etree.HTML(html_str)
        pics_url = html.xpath("//a[@class='iusc']/@m")

        for pic_url in pics_url:
            pic_url = json.loads(pic_url)['turl']
            res = requests.get(pic_url, headers=headers).content

            with open('{}.jpg'.format(count), "wb") as f:
                f.write(res)

            print('第{}张已下载完成'.format(count))

            count += 1
            time.sleep(0.5)

        first += 35
        relp += 35

        if first > 35 * 5:
            break

    except Exception as e:
        print(e)