妹子图爬取__RE__BS4

妹子图爬取

页面链接

感谢崔大佬: 原文链接

正则实现代码:

import requests
import re
import os
import random


class mzitu():

    def __init__(self):
        self.headers = {
            'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
            'Referer': 'http://i.meizitu.net'}
        self.folder_path = str(os.getcwd())

    def all_url(self, url):
        html = self.request(url).text
        all_a_re = r'<ashref="(http://www.mzitu.com/d+)"starget="_blank"'
        pattern = re.compile(all_a_re)
        all_a_list = re.findall(pattern, html)
        # for href in all_a_list:
        #     self.html(href)
        self.html(random.choice(all_a_list))

    def html(self, href):
        html = self.request(href).text
        title_pattern = re.compile(r'<h2sclass="main-title">(.*?)</h2>', re.S)
        title = re.findall(title_pattern, html)
        self.mkdir(str(title))
        max_pattern = re.compile(r'<span>(d*?)</span>', re.S)
        max_num = re.findall(max_pattern, html)
        max_num = int(max_num[-1])
        for i in range(max_num):
            page_url = href + "/" + str(i)
            self.img(page_url)

    def mkdir(self, folder):
        path = str(folder.strip())
        os.makedirs(os.path.join(self.folder_path, path))
        print(str(folder) + "------已创建")
        os.chdir(self.folder_path + str("/") + path)

    def img(self, page_url):
        img_html = self.request(page_url).text
        img_url_re = r'<imgssrc="(.*?)"salt=".*?"'
        pattern = re.compile(img_url_re, re.S)
        img_all = re.findall(pattern, img_html)
        img_url = img_all[0]
        self.save(img_url)

    def save(self, img_url):
        name = img_url[-6:]
        print(name + "正在下载")
        img = self.request(img_url)
        f = open(name, 'wb')
        f.write(img.content)
        f.close()

    def request(self, url):  # 这个函数获取网页的response 然后返回
        response = requests.get(url, headers=self.headers)
        return response


Mzitu = mzitu()  # 实例化
Mzitu.all_url('http://www.mzitu.com/all')

BS4实现代码:

import requests
from bs4 import BeautifulSoup
import os
import random


class mzitu():

    def __init__(self):
        self.headers = {
            'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
            'Referer': 'http://i.meizitu.net'}
        self.folder_path = str(os.getcwd())

    def all_url(self, url):
        html = self.request(url)  # 调用request函数把套图地址传进去会返回给我们一个response
        all_a = BeautifulSoup(html.text, 'lxml').find(
            'div', class_='all').find_all('a')

        # for i in range(1, len(all_a)):
        #     a = all_a[i]
        a = random.choice(all_a)
        title = a.get_text()
        print(u'开始保存：', title)  # 加点提示不然太枯燥了
        # 我注意到有个标题带有 ？  这个符号Windows系统是不能创建文件夹的所以要替换掉
        path = str(title).replace("?", '_')
        self.mkdir(path)  # 调用mkdir函数创建文件夹！这儿path代表的是标题title哦！！！！！不要糊涂了哦！
        href = a['href']
        # 调用html函数把href参数传递过去！href是啥还记的吧？ 就是套图的地址哦！！不要迷糊了哦！
        self.html(href)

    def html(self, href):  # 这个函数是处理套图地址获得图片的页面地址
        html = self.request(href)
        self.headers['referer'] = href
        max_span = BeautifulSoup(html.text, 'lxml').find(
            'div', class_='pagenavi').find_all('span')[-2].get_text()
        for page in range(1, int(max_span) + 1):
            page_url = href + '/' + str(page)
            self.img(page_url)  # 调用img函数

    def img(self, page_url):  # 这个函数处理图片页面地址获得图片的实际地址
        img_html = self.request(page_url)
        img_url = BeautifulSoup(img_html.text, 'lxml').find(
            'div', class_='main-image').find('img')['src']
        self.save(img_url)

    def save(self, img_url):  # 这个函数保存图片
        name = img_url[-6:]
        print(name + "正在下载")
        img = self.request(img_url)
        f = open(name, 'ab')
        f.write(img.content)
        f.close()

    def mkdir(self, path):  # 这个函数创建文件夹
        path = path.strip()
        isExists = os.path.exists(os.path.join(self.folder_path, path))
        if not isExists:
            print(path)
            os.makedirs(os.path.join(self.folder_path, path))
            os.chdir(self.folder_path + str("/") + path)  # 切换到目录
            return True
        else:
            print(u'名字叫做', path, u'的文件夹已经存在了！')
            return False

    def request(self, url):  # 这个函数获取网页的response 然后返回
        content = requests.get(url, headers=self.headers)
        return content


Mzitu = mzitu()  # 实例化
Mzitu.all_url('http://www.mzitu.com/all')  # 给函数all_url传入参数  你可以当作启动爬虫（就是入口）