selenium实现百度图片爬取

因为是百度图片是瀑布流ajax异步上传的数据，所以这里用到抓包工具来抓取链接（fiddler）

好了直接上代码，

from selenium import webdriver
from selenium.webdriver.common.by import By
import requests,time
from queue import Queue
from urllib import request
import os,gevent
from lxml import etree




def get_img(html):
    html = html.get()

    html = etree.HTML(html)

    img_url = html.xpath('//div[@id="imgid"]/div[last()]//li/@data-objurl')
    # print(img_url)
    path = './baidupic/'
    if not os.path.exists(path):
        os.makedirs(path)

    for url in img_url:
        print(url)
        # response = requests.get(url)
        # img = response.content
        try:
            fname = url.split('/')[-1]
            request.urlretrieve(url,os.path.join(path, fname))
            print('下载成功')
        except:
            print('图片不存在')


def get_page():
    #创建数据队列
    q = Queue()

    #百度图片搜索地址
    base_url = 'https://image.baidu.com/'
    #返回浏览器对象
    browser = webdriver.Chrome(executable_path=r'C:UserszhaozhiDesktopchromedriver.exe')
    #模拟访问
    browser.get(base_url)
    #输入搜索关键字
    browser.find_element_by_id('kw').send_keys('美女')
    #按键
    browser.find_element_by_class_name('s_search').click()
    # time.sleep(2)
    for i in range(10):
        browser.execute_script('window.scrollTo(0,document.body.scrollHeight)')
        # time.sleep(2)
        # html = browser.page_source


        q.put(browser.page_source)
    # browser.close()
    # print(browser.page_source)
    g_list=[]
    for i  in range(20):
        g= gevent.spawn(get_img,q)
        g_list.append(g)

    gevent.joinall(g_list)








# browser.save_screenshot('baidupic.png')
# print(browser.page_source)
# browser.find_element(By_)

if __name__ == '__main__':
    get_page()