【Python学习之旅】---Scrapt框架-实战爬取阳关问政平台多个信息(完整版）

 

 

写完之后在main.py所在的路径下执行命令即可：main.py

源代码：

items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class SunspiderItem(scrapy.Item):
    #每个帖子的标题
    title=scrapy.Field()
    #每个帖子的url
    url=scrapy.Field()
    #每个帖子的内容
    content=scrapy.Field()

sun.py

# -*- coding: utf-8 -*-
import scrapy
from sunSpider.items import SunspiderItem

class SunSpider(scrapy.Spider):
    name = 'sun'
    allowed_domains = ['wz.sun0769.com']
    url='http://wz.sun0769.com/political/index/politicsNewest?id=1&page='
    offset=1
    start_urls = [url+str(offset)]

    def parse(self, response):
        #取出每个链接的列表
        links=response.xpath("//div/ul[@class='title-state-ul']/li/span/a[@class='color-hover']/@href").extract()
        #发送每个帖子的请求，使用parse_item方法进行处理

        for link in links:
            link=response.urljoin(link) #获取绝对路径URL
            yield scrapy.Request(link,callback=self.parse_item) #回调parse_item函数

        #设置自动翻页
        if soffset<6:

            self.offset+=1
            #重新发送新的页面
            yield scrapy.Request(self.url+str(self.offset),callback=self.parse)

    def parse_item(self,response):
        item=SunspiderItem()
        #url
        item["url"]=response.url #每个帖子的URL
        #标题
        item["title"]=response.xpath("//div/div[@class='mr-three']/p[@class='focus-details']/text()").extract()[0]
        #内容
        item["content"]="".join(response.xpath("//div/div[@class='mr-three']/div[@class='details-box']/pre/text()").extract())
        yield item

pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


class SunspiderPipeline(object):
    def __init__(self):
        self.file=open("sun.txt","a",encoding="utf-8")
    def process_item(self, item, spider):
        #构造每个写入的item
        content=str(item)+'
'
        self.file.write(content)
        return item
    def spider_closed(self,spider):
        self.file.close()

最后在settings.py文件中取消管道文件的注释

ITEM_PIPELINES = {
   'sunSpider.pipelines.SunspiderPipeline': 300,
}