Python的scrapy之爬取豆瓣影评和排名

基于scrapy框架的爬影评

爬虫主程序：

import scrapy
from ..items import DoubanmovieItem

class MoviespiderSpider(scrapy.Spider):
    name = 'moviespider'
    allowed_domains = ['douban.com']
    start_urls = ['http://movie.douban.com/top250']

    def parse(self, response):
        movie_items=response.xpath('//div[@class="item"]')
        for item in movie_items:
            #print(type(item))

            movie =DoubanmovieItem()
            movie['rank']=item.xpath('div[@class="pic"]/em/text()').extract()
            movie['title']=item.xpath('div[@class="info"]/div[@class="hd"]/a/span[@class="title"][1]/text()').extract()
            movie['quote'] = item.xpath(
                'div[@class="info"]/div[@class="bd"]/p[@class="quote"]/span[@class="inq"][1]/text()').extract()
            movie['star'] = item.xpath(
                'div[@class="info"]/div[@class="bd"]/div[@class="star"]/span/text()').extract()

            movie['src']=item.xpath(
                'div[@class="pic"]/a/img/@src').extract()


            yield movie
            pass

        #取下一页的地址
        nextPageURL = response.xpath('//span[@class="next"]/a/@href').extract()
        #print(nextPageURL)
        if nextPageURL:
            url = response.urljoin(nextPageURL[-1])
            #print('url', url)
            # 发送下一页请求并调用parse()函数继续解析
            yield scrapy.Request(url, self.parse, dont_filter=False)
            pass
        else:
            print("退出")
        pass

items 对象

import scrapy


class DoubanmovieItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    rank=scrapy.Field()
    title=scrapy.Field()
    quote=scrapy.Field()
    star=scrapy.Field()
    src=scrapy.Field()
    pass

pipelines 输出管道

class DoubanmoviePipeline(object):
    def process_item(self, item, spider):
        print('电影排名:{0}'.format(item['rank'][0]))
        print('电影名称:{0}'.format(item['title'][0]))
        print('电影短评:{0}'.format(item['quote'][0]))
        print('评价分数:{0}'.format(item['star'][0]))
        print('评价人数:{0}'.format(item['star'][1]))

        print('图片链接:{0}'.format(item['src']))
        print('-' * 20)

return item

在控制台输出的结果

可以通过爬出的图片链接，下载电影的剧照，这就另说了，也可以设置一个插入数据库的管道，将这些数据插入到数据库中