妹子图图片爬取

import requests
from lxml import etree
import os
from urllib import request
import mysqlhelper


myhelper = mysqlhelper.MysqlHelper()
sql = 'insert into meizitu(name,pic_url) values(%s,%s)'

base_url = 'http://www.mzitu.com/page/%s/'
headers = {

    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
}
for i in range(1,3):
    url = base_url % i

    response = requests.get(url,headers=headers)
    html_ele = etree.HTML(response.text)

    a_list = html_ele.xpath('//ul[@id="pins"]/li/a/@href')


    for url_list in a_list:
        response = requests.get(url_list,headers)
        html_ele = etree.HTML(response.text)

        page = html_ele.xpath('//div[@class="pagenavi"]/a[5]/span')[0].text

        for pg in range(1,int(page)+1):
            image_url = str(url_list) +'/' + str(pg)
            image_name=  'meizitu/' + image_url.split('/')[-2] + image_url.split('/')[-1] + '.jpg'

            data = (image_name,image_url)
            myhelper.execute_modify_sql(sql, data)


            # if not os.path.exists('meizitu'):
            #     os.mkdir('meizitu')
            #
            #     filename = 'meizitu/' + image_url.split('/')[-2] + image_url.split('/')[-1] + '.jpg'
            #     print(filename)
            #     request.urlretrieve(image_url, filename)
            #
            # else:
            #     print('图片接收失败')