爬取58同城二手房数据存储到redis数据库和mysql数据库

主程序代码：

import scrapy
from  scrapyDemo.items  import  ScrapydemoItem

class PostSpider(scrapy.Spider):
    name = 'home'
    # allowed_domains = ['www.baidu.com']
    start_urls = ['https://bj.58.com/shahe/ershoufang/e4/?PGTID=0d30000c-0142-1050-f5c4-dad0a3db3138&ClickID=1']

    def parse(self, response):
        li_list=response.xpath('//ul[@class="house-list-wrap"]/li')
        for li in li_list:
            title=li.xpath('./div[2]/h2/a/text()').extract_first()
            price=li.xpath('./div[3]/p/b/text()').extract_first()

            item=ScrapydemoItem()
            item["title"]=title
            item["price"]=price

            yield item

管道代码：

import  pymysql
from redis import  Redis

#存储到mysql数据库
class Scrapydemotomysql(object):
    #连接
    conn=None
    #游标
    cursor=None

    def open_spider(self,spider):
        self.conn = pymysql.Connect(host="127.0.0.1", port=3306, user="root", password="root", db="spider")

    def process_item(self, item, spider):
        title=item["title"]
        price=item["price"]
        sql='insert into house values ("%s","%s")'%(title,price)
        self.cursor=self.conn.cursor()
        try:
            self.cursor.execute(sql)
            self.conn.commit()
        except Exception as e:
            print(str(e))
            self.conn.rollback()
        return  item
    def close_spider(self,spider):
        self.cursor.close()
        self.conn.close()

#存储到redis数据库
class Scrapydemotoredis(object):
    #连接
    conn=None
    def open_spider(self,spider):
        self.conn = Redis(host='127.0.0.1',port=6379)

    def process_item(self, item, spider):
        title=item["title"]
        price=item["price"]
        dict={
            'title':title,
            'price':price
        }
        dict=str(dict)
        self.conn.lpush('home',dict)
        return  item