scrapy爬取当当网

春节已经临近了尾声，也该收收心了。博客好久都没更新了，自己在年前写的爬虫也该“拿”出来了。

本次爬取的目标是当当网，获取当当网所有的书籍信息。采用scrapy+mongodb来采集存储数据。开干！

起始url：

start_urls = ['http://category.dangdang.com/cp01.00.00.00.00.00-shlist.html']

 当当书籍的一级分类二级分类都很明显的展示了出来。

ok~入口找到了，当当网也没有设置反爬措施，所以可以直接的放心爬取，如果需要大规模爬取的时候可以对爬虫的请求时间设置下，不要给别人的服务器带来太多的压力。

DOWNLOAD_DELAY = 5

 ok，我直接上代码了！

items.py

class BookDangdangItem(scrapy.Item):
    # 将数据插入到moongodb中
    price = scrapy.Field()  # 价格
    type_tag = scrapy.Field()  # 所属分类
    name = scrapy.Field()  # 书籍名称
    image_url = scrapy.Field()  # 首页的图片url
    link = scrapy.Field()  # url
    star_level = scrapy.Field()  # 星
    pub_time = scrapy.Field()  # 出版时间
    publish = scrapy.Field()  # 出版社
    brief = scrapy.Field()  # 描述

    detail = scrapy.Field()  # 书籍详情 {}

 spiders.py

# -*- coding: utf-8 -*-
import time
import logging

import scrapy
from scrapy.http.cookies import CookieJar

from ..items import BookDangdangItem
from ..settings import DEFAULT_REQUEST_HEADERS


class DangdangSpider(scrapy.Spider):
    name = 'dangdang'
    allowed_domains = ['dangdang.com']
    start_urls = ['http://category.dangdang.com/cp01.00.00.00.00.00-shlist.html']
    dom = 'http://category.dangdang.com'  # 用于拼接url
    cookie_dict = {}

    def start_requests(self):
        return [scrapy.Request(url=self.start_urls[0], callback=self.parse, headers=DEFAULT_REQUEST_HEADERS)]

    def parse(self, response):
        try:
            typestr = response.meta['type']
        except(KeyError):
            typestr = ""
        types = response.xpath('//*[@id="navigation"]/ul/li[1]/div[2]/div[1]/div/span/a')  # 获取书籍分类
        tyname = response.xpath('//*[@id="navigation"]/ul/li[1]/@dd_name').extract_first()
        if types and tyname == '分类':  # 到分类终止递归
            for type in types:
                url = self.dom + type.xpath('@href').extract_first()  # 每一个书籍分类下面的url
                typestr_new = typestr + "{0}>>".format(type.xpath('text()').extract_first())  # 多级分类

                scrapy.Spider.log(self, "Find url:{0},type{1}".format(url, typestr_new), logging.INFO)  # 配置日志信息
                yield scrapy.Request(url=url, callback=self.parse, meta={'type': typestr_new},
                                     headers=DEFAULT_REQUEST_HEADERS)
        else:
            page = int(response.xpath('//*[@id="go_sort"]/div/div[2]/span[1]/text()').extract_first())  # 当前页
            all_page = int(response.xpath('//*[@id="go_sort"]/div/div[2]/span[2]/text()').extract_first().lstrip('/'))  # 总页数
            for x in range(page, all_page):  # 处理分页
                yield scrapy.Request(url=self.dom + '/pg{0}-'.format(x) + response.url.split('/')[-1],
                                     callback=self.parse_page, headers=DEFAULT_REQUEST_HEADERS,
                                     meta={'type': typestr})

    def parse_page(self, response):
        """解析列表页中书籍的信息"""
        # cookie可以不用加，我加上只是为了测试
        cookie_jar = CookieJar()
        cookie_jar.extract_cookies(response, response.request)
        for k, v in cookie_jar._cookies.items():
            for i, j in v.items():
                for m, n in j.items():
                    self.cookie_dict[m] = n.value
        # print(self.cookie_dict)

        for item in response.xpath('//*[@id="search_nature_rg"]/ul[@class="bigimg"]/li'):
            # 所有图书
            book = BookDangdangItem()
            book['price'] = float(item.xpath('./p[@class="price"]/span[1]/text()').extract_first().lstrip('¥'))
            book['type_tag'] = response.meta['type']
            book['name'] = item.xpath('./p[@class="name"]/a/text()').extract_first().strip()
            book['image_url'] = item.xpath('./a/img/@src').extract_first()
            book['link'] = item.xpath('./p[1]/a/@href').extract_first()
            book['star_level'] = int(item.xpath('./p[@class="search_star_line"]/span/span/@style').extract_first()
                                                 .split(' ')[-1].rstrip('%;'))
            try:
                book['pub_time'] = item.xpath('.//p[@class="search_book_author"]/span[2]/text()').extract_first().split('/')[-1]
            except Exception as e:
                book['pub_time'] = time.strftime("%Y-%m-%d")
            try:
                book['publish'] = item.xpath(
                    './p[@class="search_book_author"]/span[3]/a/text()').extract_first().strip()
            except Exception as e:
                book['publish'] = "暂无出版社信息"
            try:
                book['brief'] = item.xpath('./p[2]/text()').extract_first().strip()
            except Exception as e:
                book['brief'] = "暂无书籍简述"
            yield scrapy.Request(callback=self.parse_book, cookies=self.cookie_dict,
                                 headers=DEFAULT_REQUEST_HEADERS, meta={'item': book}, url=book['link'])

    def parse_book(self, response):
        """跟进url解析书籍详情"""
        book = response.meta['item']
        book['detail'] = {}
        info = response.xpath("//ul[@class='key clearfix']/li/text()").extract()
        print(info)
        for i in info:
            t = i.partition("：")
            k = t[0].replace(" ", "")
            v = t[-1]
            if v == '':
                v = "暂无详情"
            book['detail'][k] = v

        # 这个作者详情当当不同的板块有不同的取法，看了下有很多种，写起来挺麻烦的就暂时这样处理
        try:
            book['detail']['author_detail'] = response.xpath("//span[@id=‘authorIntroduction-show’]/text()")
                .extract_first().replace('
', '')
        except Exception as e:
            book['detail']['author_detail'] = "暂无作者信息"

        yield book

说明下：cookie在本次爬虫中可以不用加，我加上只是为了测试。请求头做太多的定制。

piplines.py

from scrapy.conf import settings
from scrapy import signals
from pymongo import MongoClient


class DangDangSpiderPipeline(object):
    def __init__(self):
        # 获取setting中主机名，端口号和集合名
        host = settings['MONGODB_HOST']
        port = settings['MONGODB_PORT']
        dbname = settings['MONGODB_DBNAME']
        col = settings['MONGODB_COL']

        # 创建一个mongo实例
        client = MongoClient(host=host, port=port)

        # 访问数据库
        db = client[dbname]

        # 访问集合
        self.col = db[col]

    def process_item(self, item, spider):
        data = dict(item)
        self.col.insert(data)
        return item

settings.py

USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 
              Safari/537.36 SE 2.X MetaSr 1.0'

ROBOTSTXT_OBEY = False

DOWNLOAD_DELAY = 5

COOKIES_ENABLED = False

DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en',
    "authority": " www.dangdang.com",
    "method": "GET",
    "path": "/",
    "scheme": "http",
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "accept-encoding": 'gzip, deflate, br',
    "accept-language": 'en-US,en;q=0.9',
    "referer": None,
    "upgrade-insecure-requests": 1,
    "User-Agent": 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 
                      Safari/537.36 SE 2.X MetaSr 1.0'
}    # 可以不写使用默认的

ITEM_PIPELINES = {
    'dangdangspider.pipelines.DangDangSpiderPipeline': 300,
}

# Mongodb
# 主机环回地址
MONGODB_HOST = '127.0.0.1'
# 端口号，默认27017
MONGODB_POST = 27017
# 设置数据库名称
MONGODB_DBNAME = 'dangdangs'
# 设置集合名称
MONGODB_COL = 'books'

settings.py请求头可以不用写，使用默认的就行，我之所以加上也是测试用的，同理上面写了两个user-agent。just for fun  :-)

这次爬虫其实主要的目的是，我想了解下scrapy中的cookie是如何加上的，以及怎么用，有多少中用法！scrapy中的cookie无论是在官方文档或者一些大V的博客里都很少介绍到，但是他在实际的场景中却会用到，那就是模拟登录的时候。或者向淘宝那样需要携带cookie才能访问页面的时候。关于如何使用cookie请查看我的另一篇博客