20.Scrapy日常练手

1.创建爬虫项目：

scrapy  startproject tutorial

2.创建 spider 

cd tutorial

scrapy  genspider quotes quotes.toscrape.com

如下图：

3.

quotes.py

___________________________________________________________________________

# -*- coding: utf-8 -*-
import scrapy
from tutorial.items import TutorialItem
import logging
class QuotesSpider(scrapy.Spider):
    name = 'quotes'
    allowed_domains = ['quotes.toscrape.com']
    start_urls = ['http://quotes.toscrape.com/']

    def parse(self, response):
        quotes=response.css('.quote')
        for quote in quotes:

            item=TutorialItem()
            #内容
            item['text']=quote.css('.text::text').extract_first()

            #作者
            item['author']=quote.css('.author::text').extract_first()

            #标签
            item['tags']=quote.css('.tags .tag::text').extract_first()

            yield item

        #下一页
        next=response.css('.pager .next a::attr("href")').extract_first()
        url=response.urljoin(next)
        yield scrapy.Request(url=url,callback=self.parse)

items.py
________________________________________________________________________

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class TutorialItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()


    text=scrapy.Field()
    author=scrapy.Field()
    tags=scrapy.Field()
piplines.py

_________________________________________________________________________

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from scrapy.exceptions import DropItem

import pymysql

class TutorialPipeline(object):
    # def __init__(self):
    #     self.limit=50
    # def process_item(self, item, spider):
    #     if  item['text']:
    #         if len(item['text'])>self.limit:
    #             item['text']=item['text'][0:self.limit].rstrip()+'...'
    #         return item
    #     else:
    #         return DropItem('Missing Text')
    def __init__(self):
        pass
    def open_spider(self, spider):
        self.my_conn = pymysql.connect(
            host = '192.168.113.129',
            port = 3306,
            database = 'datas',
            user = 'root',
            password = '123456',
            charset = 'utf8'
        )
        self.my_cursor = self.my_conn.cursor()

    def process_item(self,item, spider):

        dict(item)
        insert_sql = "insert into quotes(author,tags,text) values(%s,%s,%s)"
        self.my_cursor.execute(insert_sql,[item['author'],item['tags'],item['text']])
        return  item
    def close_spider(self, spider):
        self.my_conn.commit()

        self.my_cursor.close()

        self.my_conn.close()

setting.py
___________________________________________________________________________

# Obey robots.txt rules
ROBOTSTXT_OBEY = True

ITEM_PIPELINES = {
   'tutorial.pipelines.TutorialPipeline': 200,
}

代码配置完：

保存文件格式

scrapy crawl  quotes -o quotes.xml
scrapy crawl  quotes -o quotes.csv