Python的scrapy之爬取妹子图片

闲来无事，做的一个小爬虫项目

爬虫主程序：

import scrapy
from ..items import MeiziItem

class MztSpider(scrapy.Spider):
    name = 'mzt'
    allowed_domains = ['meizitu.com']
    start_urls = ['http://meizitu.com/']

    def parse(self, response):
        tags = response.xpath(".//*[@class='tags']/span/a")
        for i in tags:
            item = MeiziItem()
            tag_href = i.xpath(".//@href").extract()[0]
            tag_name = i.xpath(".//@title").extract()[0]
            item['tag_name'] = tag_name
            item['tag_href'] = tag_href
            #print(item['tag_name'])
            #yield item
            yield scrapy.Request(url=item['tag_href'], meta={'item': item}, callback=self.parse_page)

    def parse_page(self, response):

        item = response.meta['item']
        # 进入某个标签后，爬取底部分页按钮
        page_lists = response.xpath(".//*[@id='wp_page_numbers']/ul/li")
        # 获取底部分页按钮上的文字，根据文字来判断当前标签页下总共有多少分页
        page_list = page_lists.xpath('.//text()')
        # 如果当前标签页下有多个页面，则再根据第一个按钮是否为“首页”来进行再次提取，因为这里有的页面第一个按钮是首页，有的第一个按钮是“1”
        if len(page_lists) > 0:
            if page_list[0].extract() == '首页':
                page_num = len(page_lists) - 3
            else:
                page_num = len(page_lists) - 2
        else:
            page_num = 1

        # 根据当前标签页的链接，来拼成带页码的链接
        if '_' in item['tag_href']:
            index = item['tag_href'][::-1].index('_')
            href_pre = item['tag_href'][:-index]
        else:
            if page_num == 1:
                href_pre = item['tag_href'].split('.html')[0]
            else:
                href_pre = item['tag_href'].split('.html')[0] + '_'
        for i in range(1, page_num + 1):
            item = response.meta['item']
            if page_num == 1:
                href = href_pre + '.html'
            else:
                href = href_pre + str(i) + '.html'
            item['page_list'] = href
            #yield item
            yield scrapy.Request(url=item['page_list'], meta={'item': item}, callback=self.parse_album)

    def parse_album(self, response):
        albums = response.xpath(".//*[@class='pic']")
        for album in albums:
            item = response.meta['item']
            album_href = album.xpath(".//a/@href").extract()[0]
            album_name = album.xpath(".//a/img/@alt").extract()[0]
            item['album_name'] = album_name
            item['album_href'] = album_href
            #yield item
            yield scrapy.Request(url=item['album_href'], meta={'item': item}, callback=self.parse_img)

    def parse_img(self, response):
        img_list = response.xpath(".//*/p/img")
        for img in img_list:
            item = response.meta['item']
            img_title = img.xpath(".//@alt").extract()[0]
            if img_title == '':
                for i in range(1, len(img_list + 1)):
                    img_title = item['album_name'] + '_' + str(i)
            else:
                img_title = img_title
            img_urls = img.xpath(".//@src").extract()
            img_src = img.xpath(".//@src").extract()[0]
            item['img_title'] = img_title
            item['img_src'] = img_src
            item['img_urls'] = img_urls
            yield item

items设置

import scrapy


class MeiziItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # 标签名称
    tag_name = scrapy.Field()
    # 标签链接
    tag_href = scrapy.Field()
    # 进入某标签后的所有链接，加页码的
    page_list = scrapy.Field()
    # 图片专辑名称
    album_name = scrapy.Field()
    # 图片专辑链接
    album_href = scrapy.Field()
    # 照片标题
    img_title = scrapy.Field()
    # 照片链接
    img_src = scrapy.Field()
    # 照片链接集合，用于ImagesPipeline下载图片
    img_urls = scrapy.Field()

输出管道：

print('正在爬取...')
print('老湿机，请耐心等待哟...')
class MeiziPipeline(object):
    def process_item(self, item, spider):
        print('标签名称:',item['tag_name'])
        print('标签链接:',item['tag_href'])
        print('页码:',item['page_list'])
        print('图片专辑名称:',item['album_name'])
        print('图片专辑链接:',item['album_href'])
        print('照片标题:',item['img_title'])
        print('照片链接:',item['img_src'])
        print('照片链接集合:',item['img_urls'])
        print('----------------')
        return item

保存到本地的管道：

import scrapy
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem


class MztImagesPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        for image_url in item['img_urls']:
            yield scrapy.Request(image_url)

    def item_completed(self, results, item, info):
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem("该Item没有图片")
        return item

setting的设置，往setting.py加入

BOT_NAME = 'meizi'

SPIDER_MODULES = ['meizi.spiders']
NEWSPIDER_MODULE = 'meizi.spiders'

IMAGES_STORE = r'G:\mzt'    # 图片存储路径
IMAGES_EXPIRES = 90             # 过期天数
IMAGES_MIN_HEIGHT = 100         # 图片的最小高度
IMAGES_MIN_WIDTH = 100          # 图片的最小宽度

爬取的最终结果

本来想把这些图片分门分类的保存，然而不太会，所有的图片全保存在一个文件夹下面，，，