爬虫_斗图啦_表情包下载

为下一个多线程练练手

import requests
from lxml import etree
import re
from urllib import request
import time
import os

def get_html(url):
    headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'}
    try:
        response = requests.get(url, headers=headers)
        response.raise_for_status()

        response.encoding = 'utf-8'
        return response.text
    except:
        return 0


def parse_html(html):
    html_element = etree.HTML(html)
    imgs = html_element.xpath('//div[@class="page-content text-center"]//img[@class!="gif"]')
    for img in imgs:
        url = img.get('data-original')
        name = img.get('alt')
        name = re.sub('[?？，。]', '', name)#看系统的命名要求了
        tail = os.path.splitext(url)[1] #取后缀

        file_name = name + tail  #拼接文件名
        request.urlretrieve(url, 'images/' + file_name)  #注意提前新建images文件夹


def main():
    for i in range(1, 50):
        url = 'http://www.doutula.com/photo/list/?page=%d' % i
        html = get_html(url)
        while html == 0:
            time.sleep(2)
            html = get_html(url)
        parse_html(html)
        # break

            
if __name__ == '__main__':
    main()

不多说了，沙海开始了。