使用scrapy 爬取酷狗音乐歌手及歌曲名并存入mongodb中

备注还没来得及写，共爬取八千多的歌手，每名歌手平均三十首歌曲算，大概二十多万首歌曲

run.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
__author__ = 'Zqf'
from dingdian_simple.spiders.dingdian_spider import DingdianSimpleSpider
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

# 获取settings.py模块的设置
settings = get_project_settings()
process = CrawlerProcess(settings=settings)

# 可以添加多个spider
process.crawl(DingdianSimpleSpider)

# 启动爬虫，会阻塞，直到爬取完成
process.start()

kugou.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re

__author__ = 'Zqf'

import scrapy
from kugoumusic.items import KugoumusicItem
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Rule


class KugouSpiders(scrapy.spiders.CrawlSpider):
    name = 'kugou'
    
    start_urls = ['http://www.kugou.com/']

    rules = (
        Rule(LinkExtractor(allow=['http://www.kugou.com/yy/html/singer.html',
                                  'http://www.kugou.com/yy/singer/index/d-([a-z]|null)-1.html'])),
        Rule(LinkExtractor(allow=['http://www.kugou.com/yy/singer/home/d+.html']), callback='parse_item')
    )
    
    def parse_item(self, response):
        singer = response.xpath('//div/div[@class="clear_fix"]/strong/text()').extract_first()
        print(singer)
        songs = response.xpath('//ul[@id="song_container"]/li//span[@class="text"]/i/text()').extract()
        print(songs)
    
        item = KugoumusicItem()
        item['singer'] = singer
        item['songs'] = songs
        
        yield item

items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class KugoumusicItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    singer = scrapy.Field()
    songs = scrapy.Field()

pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from pymongo import MongoClient


class KugoumusicPipeline(object):
    
    def open_spider(self, spider):
        # mongo_config = spider.settings['MONGO_CONFIG']
        # host = '127.0.0.1', port = 27017
        self.client = MongoClient(host='127.0.0.1', port=27017)
        self.coll = self.client['student_db']['kugou']
        self.li = []
        
    def close_spider(self, spider):
        self.insert()
        self.client.close()
        
    def insert(self):
        self.coll.insert_many(self.li)
    
    def process_item(self, item, spider):
        if len(self.li) >= 100:
            self.insert()
            self.li = []
            print("成功插入100条数据-------------------------------------")
        else:
            self.li.append(dict(item))
        
        return item

settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for kugoumusic project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'kugoumusic'

SPIDER_MODULES = ['kugoumusic.spiders']
NEWSPIDER_MODULE = 'kugoumusic.spiders'

# MONGO_CONFIG = ['192.168.62.35:1806, '
#               '192.168.62.240:1806, '
#               '192.168.62.23:1806, '
#               '192.168.62.32:1806, '
#               '192.168.62.25:1806, '
#               '192.168.62.28:1806, '
#               '192.168.62.241:1806']

# MONGO_CONFIG = {
#     'host': '127.0.0.1',
#     'port': 27017
    # 'user': 'root',
    # 'password': '123456',
    # 'db': 's1806',
    # 'charset': 'utf8'
# }
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'kugoumusic (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    'Connection': 'keep-alive',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9',
}

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'kugoumusic.middlewares.KugoumusicSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'kugoumusic.middlewares.KugoumusicDownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'kugoumusic.pipelines.KugoumusicPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'