scrapy数据增量式爬取

爬取过的数据跳过
1、通过url判断
2、通过数据指纹判断

创建爬虫项目 ：scrapy startproject  xxx

cd xxx

创建爬虫文件：scrapy genspider -t crawl spidername www.xxx.com

一、根据url判断

爬虫文件

# -*- coding: utf-8 -*-
import scrapy
from redis import Redis
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from increment1.items import Increment1Item
"""
爬取过的数据跳过
1、通过url判断
2、通过数据指纹判断
"""


class FirstSpider(CrawlSpider):
    name = 'first'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.4567tv.tv/index.php/vod/show/id/7.html']

    rules = (
        Rule(LinkExtractor(allow=r'/index.php/vod/show/id/7/page/d+.html'), callback='parse_item', follow=True),
    )

    def parse_detail(self, response):
        item = response['item']
        actor = response.xpath('//div[@class="stui-content__detail"]/p[3]//text()').extract_first()
        item['actor'] = actor
        yield item

    def parse_item(self, response):
       
        conn = Redis(host='127.0.0.1', port=6388)
        detail_url_list = "https://www.4567tv.tv" + response.xpath('//li[@class="col-md-6 col-sm-4 col-xs-3"]/div/a/@href').extract_first()
        for url in detail_url_list:
            item = Increment1Item()
            ex = conn.sadd('movies_url', url)
            if ex == 1:   # 说明redis里没有该url
                yield scrapy.Request(url=url, callback=self.parse_detail, meta={"item", item})
            else:
                print('爬过的数据不在爬取！！！')

在管道文件里进行存储

from redis import Redis


class Increment1Pipeline(object):
    def open_spider(self, spider):
        self.conn = Redis(host='127.0.0.1', port=6388)

    def process_item(self, item, spider):
        print('新数据写入')
        dic = {
            'actor': item['actor']
        }
        self.conn.lpush('move_data', dic)

二、根据数据进行指纹识别

爬虫文件

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from incerment2.items import Incerment2Item
import hashlib
from redis import Redis


class FirstSpider(CrawlSpider):
    name = 'first'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.qiushibaike.com/text/']

    rules = (
        Rule(LinkExtractor(allow=r'/text/page/d+/'), callback='parse_item', follow=True),
    )

    def parse_item(self, response):
        self.conn = Redis(host='127.0.0.1', port=6388)

        div_list = response.xpath('//div[@class="content-left"]/div')
        for div in div_list:
            item = Incerment2Item()
            item['author'] = div.xpath('./div[1]/a[2]/h2/text()').extract_first()
            content = div.xpath('./a[1]/div/span/text()').extract()
            content = ''.join(content)
            item['content'] = content

            # 将当前爬取的数据做哈希唯一标识(数据指纹)
            source = item['author']+item['content']
            hashValue = hashlib.sha256(source.encode()).hexdigest()

            ex = self.conn.sadd('hashValue', hashValue)
            if ex == 1:
                yield item

            else:
                print('数据未更新')