scrapy爬取简书整站文章

在这里我们使用CrawlSpider爬虫模板， 通过其过滤规则进行抓取， 并将抓取后的结果存入mysql中，下面直接上代码：

jianshu_spider.py

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from jianshu.items import JianshuItem
import html


class JianshuSpiderSpider(CrawlSpider):
    name = 'jianshu_spider'
    allowed_domains = ['jianshu.com']
    start_urls = ['http://jianshu.com/']

    rules = (
        Rule(LinkExtractor(allow=r'.*/p/[0-9a-z]{12}.*'), callback='parse_article', follow=True),
    )

    def parse_article(self, response):
        article_code = response.url.split("?")[0].split("/")[-1]
        title = response.xpath('//h1[@class="title"]/text()').get().strip()
        author = response.xpath('//div[contains(@class, "author")]/div[@class="info"]//span[@class="name"]/a/text()').get().strip()
        head_img = response.xpath('//div[contains(@class, "author")]/a[@class="avatar"]/img/@src').get()
        pub_time = response.xpath('//span[@class="publish-time"]/text()').get().strip().replace('*','')
        head_img_url = "http:{}".format(head_img)
        # 存储到数据库中，需要对‘/’转义
        # content = html.escape(response.xpath('//div[@class="show-content"]').get())
        content = response.xpath('//div[@class="show-content"]').get()

        yield JianshuItem(
            article_code = article_code,
            title = title,
            author = author,
            head_img_url = head_img_url,
            content = content,
            pub_time = pub_time,)

items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class JianshuItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    article_code = scrapy.Field()
    title = scrapy.Field()
    author = scrapy.Field()
    pub_time = scrapy.Field()
    head_img_url = scrapy.Field()
    content = scrapy.Field()

pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from jianshu import model

class JianshuPipeline(object):

    def __init__(self):
        self.session = model.DBSession()

    def process_item(self, item, spider):
        # 这里的item属于字典类型
        article = model.Article(**item)
        try:
            self.session.add(article)
            self.session.commit()
        except Exception as e:
            print("="*100)
            print("INSERT ERROR!")
            self.session.rollback()
        return item
    def open_spider(self, spider):
        pass

    def close_spider(self, spider):
        self.session.close()

model.py

from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import String, Text, Time, Column, Integer, VARCHAR
from sqlalchemy.orm import sessionmaker

# 创建数据库链接接口
engine = create_engine("mysql+pymysql://jianshu:jianshu@localhost:3306/jianshu?charset=utf8mb4", echo=False)

# 声明映像， 即实际数据库表的基本准则的映射类
# 其维持类和数据库表关系目录
Base = declarative_base()

class Article(Base):

    __tablename__ = "jianshu_article"

    id = Column(Integer, autoincrement=True, primary_key=True)
    article_code = Column(String(16), nullable=False)
    title = Column(Text)
    author = Column(String(16))
    pub_time = Column(Time)
    head_img_url = Column(VARCHAR(256))
    content = Column(Text)

DBSession = sessionmaker(bind=engine)

if __name__ == '__main__':
    Base.metadata.create_all(engine)