爬虫 ——（50页）books

代码如下所示

import scrapy
from scrapy.selector.unified import SelectorList
from bookspider.items import BooksItem
class BooksSpider(scrapy.Spider):
    name = 'books'
    allowed_domains = ['books.toscrape.com']
    start_urls = ['http://books.toscrape.com/catalogue/page-1.html']
    url = "http://books.toscrape.com"
    def parse(self, response):
        # print('*')
        # print(type(response))    #  <class 'scrapy.http.response.html.HtmlResponse'>
        # print('*')

        Books_lis = response.xpath("//div/ol[@class='row']/li")
        # print('*')
        # print(type(Books_lis))
        # print('*')
        for Books_li in Books_lis:
            book_name = Books_li.xpath(".//h3/a/@title").getall()
            book_price = Books_li.xpath(".//div[@class='product_price']/p[@class='price_color']/text()").getall()

            book_price = "".join(book_price)
            item = BooksItem(book_name=book_name,book_price=book_price)
            # books_all = {"book_name":book_name,"book_price":book_price}
            # yield books_all
            yield item

        next_url = response.xpath("//ul[@class = 'pager']/li[@class = 'next']/a/@href").get()
        #
        if  next_url:
            # 如果找到下一页的URL，就得到绝对路径，构造新的Request对象
            next_url = response.urljoin(next_url)
            yield scrapy.Request(next_url,callback=self.parse)

        # if  not next_url:  #此方法报错  待解决
        #     return
        # else:
        #     yield scrapy.Request(self.url + next_url, callback=self.parse)