scrapy 图片爬取 多层多页 保存不同的文件夹 重命名full文件夹

记录下整个爬虫代码，我已经把实验网站爬完了。。

items.py

import scrapy


class DemoItem(scrapy.Item):
    # define the fields for your item here like:
    folder_name = scrapy.Field()  #选取页面里的主题或者标题作为文件夹名字，代替full文件夹
    #img_name = scrapy.Field()  # 提取的图片的名字，如果没有就不使用
    img_url = scrapy.Field()  # 图片的链接

spider.py

# -*- coding: utf-8 -*-
import scrapy
from demo.items import DemoItem


class LogosSpider(scrapy.Spider):
    name = 'logos'
    allowed_domains = ['tttt8.net']
    #start_urls = ['http://www.tttt8.net/category/legbaby/']
    #start_urls = ['http://www.tttt8.net/category/ugirls/']
    #start_urls = ['http://www.tttt8.net/category/kelagirls/']
    start_urls = ['http://www.tttt8.net/category/xiurenwang/micatruisg/']
    #这里的page是用来构造一级页面翻页链接
    page = 1

    def parse(self, response):
        #提取所有专栏的列表
        li_list = response.xpath('//*[@id="post_container"]/li')
        for li in li_list:
            #实例化
            item = DemoItem()
            #这里只提取专栏的名字，用于后面储存文件夹的命名
            item['folder_name'] = li.xpath('./div[2]/h2/a/text()').extract_first()
            #提取二级页面的链接，准备给二级页面函数去工作
            next_plink = li.xpath('./div[1]/a/@href').extract_first()
            #item把一级页面实例化的内容传送给二级页面接收
            yield scrapy.Request(url = next_plink, callback = self.parse2, meta = {'item':item})
        # 一级页面的翻页列表/ 自行查找别的方法翻页，网上好多，我觉得这个构造更简单
        page_list = response.xpath('//div[@class="pagination"]/a/@href').extract()
        #找到最后一页的页码
        last_page = page_list[-1]
        #提取出最大的页码
        max_num = int(last_page[-2])
        #构造翻页的页码链接
        if self.page <= max_num:
            self.page += 1
            new_page_url = self.start_urls[0] + 'page/' + str(self.page) + '/'
            yield scrapy.Request(url = new_page_url, callback = self.parse)

    def parse2(self, response):
        #接收一级页面的内容，和翻页请求的衔接，不然翻页就丢失了，会报错
        item = response.meta['item']
        p_list = response.xpath('//*[@id="post_content"]/p/img')
        #正常的提取图片的链接给pepeline去下载
        for img in p_list:
            img_url = img.xpath('./@src').extract_first()
            # 这里必须加中括号[]，图片下载的函数，要求是list类型
            item['img_url'] = [img_url]
            yield item
        # 二级页面的翻页列表，然后在后面yield请求
        next_page_list = response.xpath('//*[@id="content"]/div[1]/div[3]/div[2]/a/@href').extract()
        for next_page in next_page_list:
            #这里一定要加meta，不然二级的翻页就error，这里我搞了好久才发现。
            yield scrapy.Request(url = next_page, callback = self.parse2, meta = {'item':item})

settings.py

# -*- coding: utf-8 -*-


BOT_NAME = 'demo'
SPIDER_MODULES = ['demo.spiders']
NEWSPIDER_MODULE = 'demo.spiders'

#存储路径和header
IMAGES_STORE = 'D:pics'
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'

DOWNLOAD_DELAY = 0.2
#机器人关掉
ROBOTSTXT_OBEY = False

ITEM_PIPELINES = {
    'demo.pipelines.DemoPipeline': 300,
}
#下载指定的field
IMAGES_URLS_FIELD = 'img_url'

pipelines.py

import scrapy
from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline


class DemoPipeline(ImagesPipeline):
    # 固定的改写的函数，不需要修改
    def get_media_requests(self, item, info):
        for img_url in item['img_url']:
            # 下载完后给别的函数去改名字，所以用meta传下去
            yield scrapy.Request(img_url, meta = {'item':item})

    def file_path(self, request, response = None, info = None):
        item = request.meta['item']
        folder_name = item['folder_name']
        # img_name = item['img_name']  #图片没有名字不启用这个语句
        # 因为图片没有名字就用url截取最后的字符串作为名字
        image_guid = request.url.split('/')[-1]
        img_name = image_guid
        # name = img_name + image_guid
        # name = name + '.jpg'
        # 0 代表文件夹，1 代表文件
        filename = u'{0}/{1}'.format(folder_name, img_name)
        return filename

    # 固定改写的函数，不需要修改
    def item_completed(self, results, item, info):
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem('Image Downloaded Failed')
        return item

结果：