scrapy爬取中关村在线手机频道

# -*- coding: utf-8 -*-
import scrapy
from pyquery import PyQuery as pq

from zolphone.items import ZolphoneItem


class PhoneSpider(scrapy.Spider):
    name = "phone"
    # allowed_domains = ["www.zol.com.cn"]
    # start_url = 'http://detail.zol.com.cn/cell_phone_index/subcate57_0_list_1_0_1_1_0_1.html'
    start_url = 'http://detail.zol.com.cn/cell_phone_index/subcate57_0_list_1_0_1_1_0_'

    def start_requests(self):

        for page in range(1, 209):
            url = self.start_url + str(page) + '.html'
            yield scrapy.Request(url,callback=self.parse_index)


    def parse_index(self, response):
        base_url = 'http://detail.zol.com.cn'
        doc = pq(response.text)
        lis = doc('.list-box .list-item').items()
        for result in lis:
            detail_url = base_url + result.find('.pro-intro h3 a').attr('href')
            yield scrapy.Request(url=detail_url, callback=self.parse_detail)

    def parse_detail(self,response):
        doc = pq(response.text)
        title1 = response.css('.page-title h1::text').extract_first()
        title2 = doc('.page-title h2').text()
        price = doc('.product-price .price-type').text()
        release_time = doc('.section div h3 .showdate').text()
        print(title1, title2, price, release_time)
        item = ZolphoneItem()
        item['title1'] = title1
        item['title2'] = title2
        item['price'] = price
        item['release_time'] = release_time

        yield item

import scrapy


class ZolphoneItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title1 = scrapy.Field()
    title2 = scrapy.Field()
    price = scrapy.Field()
    release_time = scrapy.Field()