爬取二重网页

1.用 scrapy 新建一个 sun0769 项目

scrapy startproject sun0769

2.在 items.py 中确定要爬去的内容

import scrapy


class Sun0769Item(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    problem_type = scrapy.Field()
    title = scrapy.Field() 
    number = scrapy.Field() 
    content = scrapy.Field() 
    Processing_status = scrapy.Field()
    url = scrapy.Field() 

3.快速创建 CrawlSpider模板

scrapy genspider -t crawl dongguan wz.sun0769.com

注意  此时中的名称不能与项目名相同

4.打开 dongguan.py 编写代码

# -*- coding: utf-8 -*-
# 导入scrapy 模块
import scrapy
# 导入匹配规则类，用来提取符合规则的链接
from scrapy.linkextractors import LinkExtractor
# 导入CrawlSpiderl类和Rule
from scrapy.spiders import CrawlSpider, Rule
# 导入items中的类
from sun0769.items import Sun0769Item

class DongguanSpider(CrawlSpider):
    name = 'dongguan'
    allowed_domains = ['wz.sun0769.com']
    start_urls = ['http://d.wz.sun0769.com/index.php/question/huiyin?page=30']
    pagelink = LinkExtractor(allow=r"page=d+")
    pagelink2 = LinkExtractor(allow=r"/question/d+/d+.shtml")

    rules = (
        Rule(pagelink, follow=True ),
        Rule(pagelink2, callback='parse_item',follow=True ),

    )

    def parse_item(self, response):
        #print response.url 
        item = Sun0769Item() 
        # xpath 返回是一个列表
        #item['problem_type'] = response.xpath('//a[@class="red14"]').extract()
        item['title'] = response.xpath('//div[@class="pagecenter p3"]//strong[@class="tgray14"]/text()').extract()[0].split(" ")[-1].split(":")[-1]
        # item['title'] = response.xpath('//div[@class="pagecenter p3"]//strong[@class="tgray14"]/text()').extract()[0]
        item['number'] = response.xpath('//div[@class="pagecenter p3"]//    strong[@class="tgray14"]/text()').extract()[0].split("：")[1].split("  ")[0]
        #item['content'] = response.xpath().extract()
        #item['Processing_status'] = response.xpath('//div/span[@class="qgrn"]/text()').extract()[0]
        # 把数据传出去
        yield item
        
        

5.在piplines.py写代码

 1 # -*- coding: utf-8 -*-
 2 
 3 # Define your item pipelines here
 4 #
 5 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 6 # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
 7 
 8 import json
 9 
10 class TencentPipeline(object):
11     def open_spider(self, spider):
12         self.filename = open("dongguan.json", "w")
13 
14     def process_item(self, item, spider):
15         text = json.dumps(dict(item), ensure_ascii = False) + "
"
16         self.filename.write(text.encode("utf-8")
17         return item
18 
19     def close_spider(self, spider):
20         self.filename.close()
复制代码

6.在setting.py设置相关内容

---

问题:

1.怎么把不同页面的内容整合到一块

2.内容匹配还有些困难（xpath，re）