scrapy框架爬取全书网一类书籍名、作者名和简介

1.  创建一个工程项目：scrapy startproject quanshuwang

2.  cd quanshuwang  (命令行下cd到项目目录下）

　  scrapy genspider quanshu www.quanshuwang.com        在spiders下会生成一个quanshu.py文件　　　　　　(scrapy genspider 爬虫文件的名称 起始url)

3.   改settings.py

　　

USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
ROBOTSTXT_OBEY = False

　　存本地、存数据库需要改：

     

ITEM_PIPELINES = {
   'quanshuwang.pipelines.QuanshuwangPipeline': 300,
   'quanshuwang.pipelines.MysqlSpider': 400,
}

4.   爬虫文件quanshu.py

　　

# -*- coding: utf-8 -*-
import scrapy
from quanshuwang.items import QuanshuwangItem

class QuanshuSpider(scrapy.Spider):
    name = 'quanshu'
    # allowed_domains = ['www.quanshuwang.com/list/0_1.html']
    start_urls = ['http://www.quanshuwang.com/list/0_1.html']

    def parse(self, response):
        try:
            li_list = response.xpath('//ul[@class="seeWell cf"]/li')
            # print(111)
            # print(li_list)
            for li in li_list:
                item = QuanshuwangItem()
                item['book_name'] = li.xpath('./span/a/text()').extract_first()
                item['author'] = li.xpath('./span/a[2]/text()').extract_first()
                item['desc'] = li.xpath('./span/em/text()').extract_first()  # 当简介较多时，会有个‘更多’两字，可以不要了

                yield item
            next_page = response.xpath('//a[@class="next"]/@href').extract_first()
            if next_page:
                yield scrapy.Request(next_page, callback=self.parse)
        except Exception as e:
            print(e)

5.  items.py文件

　　book_name,author,desc分别与我数据库的字段对应

class QuanshuItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    book_name = scrapy.Field()
    author = scrapy.Field()
    desc = scrapy.Field()

6.   pipelines.py 管道文件

　　

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html


# class QuanshuwangPipeline(object):
#     f = None
#
#     def open_spider(self, spider):
#         self.f = open('book_list.txt', 'a+', encoding='utf-8')
#
#     def process_item(self, item, spider):
#         try:
#             print('%s 正在下载...' % item['book_name'])
#             book_name =item['book_name']
#             author = item['author']
#             desc = item['desc']
#             self.f.write('《'+book_name+'》' + '
' +' 作者：' +author + '
' +' 简介：'+ desc.strip()+'

')
#         except Exception as e:
#             print(e)
#         return item
#     def close_spider(self, spider):
#         self.f.close()


#将数据存储到mysql数据库中
import pymysql
class MysqlSpider(object):
    conn = None
    cursor = None
    #开始爬虫
    def open_spider(self,spider):
        #连接数据库
        self.conn = pymysql.connect(host='172.16.25.37',port=3306,user='root',password='root',db='scrapy')
        # 获取游标指针
        self.mycursor = self.conn.cursor()

    #执行爬虫
    def process_item(self,item,spider):
        #sql语句
        book_name = item['book_name']
        author = item['author']
        desc = item['desc']
        print('《'+book_name + '》'+' 正在下载...')
        sql = "insert into quanshuwang VALUES (null,'%s','%s','%s')"%(book_name,author,desc)

        try:
            self.mycursor.execute(sql)
            self.conn.commit()
            # print(111)
        except Exception as e:
            print(e)
            # print(222)
            self.conn.rollback()

        return item

    #结束爬虫
    def close_spider(self,spider):
        #关闭游标和连接
        self.mycursor.close()
        self.conn.close()

　

 pycharm:

数据库: