Scrapy爬取IT之家

创建项目

scrapy startproject ithome

创建CrawSpider

scrapy genspider -t crawl IT ithome.com

items.py

import scrapy


class IthomeItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    content = scrapy.Field()

it.py

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from ithome.items import IthomeItem

class ItSpider(CrawlSpider):
    name = 'it'
    allowed_domains = ['ithome.com']
    start_urls = ['https://it.ithome.com/ityejie/']

    rules = (
        Rule(LinkExtractor(allow=r'ityejie/'), follow=True),
        Rule(LinkExtractor(allow=r'html/it/d+.htm', restrict_xpaths='//*[@id="wrapper"]//*[@class="block"]'), callback='parse_item', follow=True),
    )

    def parse_item(self, response):
        i = {}
        #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
        #i['name'] = response.xpath('//div[@id="name"]').extract()
        #i['description'] = response.xpath('//div[@id="description"]').extract()

        i['title'] = response.xpath('//*[@id="wrapper"]/div[1]/div[2]/h1/text()').extract()[0]
        i['content'] = response.xpath('//*[@id="paragraph"]/p/text()').extract()
        yield i

pipelines.py

import json

class IthomePipeline(object):


    def __init__(self):
        self.filename = open("it.json", "w")

    def process_item(self, item, spider):
        text = json.dumps(dict(item), ensure_ascii = False) + ",
"
        self.filename.write(text)
        return item

    def close_spider(self, spider):
        self.filename.close()

settings.py

BOT_NAME = 'ithome'

SPIDER_MODULES = ['ithome.spiders']
NEWSPIDER_MODULE = 'ithome.spiders'



# Obey robots.txt rules
ROBOTSTXT_OBEY = False

ITEM_PIPELINES = {
    'ithome.pipelines.IthomePipeline': 300,
}

执行

scrapy crawl it