投诉网站爬虫

# -*- coding: utf-8 -*-
import scrapy
from yg.items import YgItem

class YgSpiderSpider(scrapy.Spider):
    name = 'yg_spider'
    allowed_domains = ['wz.sun0769.com']
    start_urls = ['http://wz.sun0769.com/index.php/question/questionType?type=4&page=0']

    def parse(self, response):
        tr_list = response.xpath("//div[@class='greyframe']/table[2]/tr/td/table/tr")
        for tr in tr_list:
            item = YgItem()
            item["title"] = tr.xpath("./td[2]/a[2]/@title").extract_first()
            item["href"] = tr.xpath("./td[2]/a[2]/@href").extract_first()
            item["update_time"] = tr.xpath("./td[last()]/text()").extract_first()
            # print(item)

            yield scrapy.Request(
                item["href"],
                callback=self.parse_detail,
                meta={"item":item}
            )

        next_url = response.xpath("//a[text()='>']/@href").extract_first()
        if next_url is not None:
            yield scrapy.Request(
                next_url,
                callback=self.parse
            )

    def parse_detail(self,response): #处理详情页
        item = response.meta["item"]
        item["content"] = response.xpath("//div[@class='c1 text14_2']//text()").extract()
        item["content_img"] = response.xpath("//div[@class='c1 text14_2']//img/@src").extract()
        item["content_img"] = ["http://wz.sun0769.com"+i for i in item["content_img"]]
        # print(item)
        yield item

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import re
import json

class YgPipeline(object):
    def process_item(self, item, spider):
        item["content"] = self.process_content(item["content"])
        with open("yg.txt", "a", encoding="utf-8") as f:
            f.write(json.dumps(dict(item), ensure_ascii=False, indent=4))
            f.write("
")
        return item

    def process_content(self, content):
        content = [re.sub(r'xa0|s',"",i) for i in content]
        content = [i for i in content if len(i)>0]
        return content

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class YgItem(scrapy.Item):
    # define the fields for your item here like:
    title = scrapy.Field()
    update_time = scrapy.Field()
    href = scrapy.Field()
    content = scrapy.Field()
    content_img = scrapy.Field()
    # pass