唯一图库爬取图片

项目一：唯一图库

项目概述：根据需要到唯一图库爬取图片

所用技术：scrapy,urllib，字符串处理，百分号格式化

爬虫程序根据setting和item配置把爬取到的数据交给pipline处理

相关操作记录如下：

爬虫程序

#!/usr/bin/env python
#-*- coding:utf-8 -*-
#s1.py
import scrapy
from scrapy.selector import HtmlXPathSelector
from spider1 import items

class LL(scrapy.spiders.Spider):
    name = 'xx'
    start_urls=['http://www.mmonly.cc/sgtp/',]
    def parse(self,response):
        hxs = HtmlXPathSelector(response)

        item = items.Spider1Item()
        item['names'] = hxs.select('//div[@class="item_t"]//img/@alt').extract()
        item['imgs'] = hxs.select('//div[@class="item_t"]//img/@src').extract()
        yield item

settings.py

# -*- coding: utf-8 -*-
#settings.py
BOT_NAME = 'spider1'
#
SPIDER_MODULES = ['spider1.spiders']
NEWSPIDER_MODULE = 'spider1.spiders'
ROBOTSTXT_OBEY = True
ITEM_PIPELINES = {
   'spider1.pipelines.Spider1Pipeline': 100,
}

items.py

# -*- coding: utf-8 -*-
#items.py
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class Spider1Item(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    names = scrapy.Field()
    imgs = scrapy.Field()

pipelines.py

# -*- coding: utf-8 -*-
#pipelines.py
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html


class Spider1Pipeline(object):
    def process_item(self, item, spider):

        names = item['names']
        imgs = item['imgs']

        print names[0],imgs[0]
        for i in range(len(names)):
            if names[i] and imgs[i]:
                img_name = names[i] + '.jpg'
                c = str(imgs[i])
                if c.startswith('h'):
                    net_url = c
                else:
                    net_url = 'http://www.xiaohuar.com' + c
                local_file = 'C:\Users\wenxianfeng\Desktop\img\%s' % img_name
                import urllib
                urllib.urlretrieve(net_url, local_file)

        return item