爬取人力资源社保局咨询问题

创建项目

scrapy startproject shebao

items.py

import scrapy


class ShebaoItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    content = scrapy.Field()
    url = scrapy.Field()
    number = scrapy.Field()

创建CrawSpider，使用模版crawl

scrapy genspider -t crawl SB www.bjrbj.gov.cn

SB.py

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from shebao.items import ShebaoItem

class SbSpider(CrawlSpider):
    name = 'SB'
    allowed_domains = ['www.bjrbj.gov.cn']
    start_urls = ['http://www.bjrbj.gov.cn/mzhd/list_more.htm?stid=-1&ps=10&ishot=0&pn=1&ps=10']

    rules = (
        Rule(LinkExtractor(allow=r'&pn=d+'), follow=True),
        Rule(LinkExtractor(allow=r'/mzhd/detail_d+.htm'), callback = 'parse_item'),
        #Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
    )

    def parse_item(self, response):

        item = ShebaoItem()
        item['title'] = response.xpath('//div[@class="xx_neirong"]/h1/text()').extract()[0]
        # 编号
        item['number'] = response.xpath('//p[@class="jz_p1"]/text()').extract()[0]

        item['content'] = response.xpath('//p[@class="jz_p2"]/text()').extract()[0]
        # 链接
        item['url'] = response.url

        yield item

pipelines.py

import json

class ShebaoPipeline(object):


    def __init__(self):
        self.filename = open("shebao.json", "w")

    def process_item(self, item, spider):
        text = json.dumps(dict(item), ensure_ascii = False) + ",
"
        self.filename.write(text)
        return item

    def close_spider(self, spider):
        self.filename.close()

settings.py

BOT_NAME = 'shebao'

SPIDER_MODULES = ['shebao.spiders']
NEWSPIDER_MODULE = 'shebao.spiders'


ITEM_PIPELINES = {
    'shebao.pipelines.ShebaoPipeline': 300,
}

LOG_FILE = "dg.log"
LOG_LEVEL = "DEBUG"

执行

scrapy crawl SB