zoukankan      html  css  js  c++  java
  • scrapy批量下载图片

    # -*- coding: utf-8 -*-
    import scrapy
    from rihan.items import RihanItem
    
    
    class RihanspiderSpider(scrapy.Spider):
        name = "rihanspider"
        # allowed_domains = ["*******"]
        start_urls = [**************']
    
        def parse(self, response):
            # print(response.text)
            for each in response.css('.img li'):
                index_url = each.css('a::attr(href)').extract_first()
                # print(index_url)
                yield scrapy.Request(url=index_url,callback=self.parse_detail)
    
            next = response.xpath('//a[contains(.,"下一页")]/@href').extract_first()
            if next:
                url = response.urljoin(next)
                yield scrapy.Request(url, callback=self.parse)
    
        def parse_detail(self, response):
    
            item = RihanItem()
            image_title = response.css('.width .weizhi h1::text').extract_first()
            item['image_title'] = image_title
    
            for each in response.css('.content'):
                # 注意这里如果想用scrapy内置的下载图片模块,这里要用extract()而不是extract_first()
                image_url = each.css('img::attr(src)').extract()
                # print(image_url)
                item['image_url'] = image_url
    
                yield item
    
            next = response.xpath('//a[contains(.,"下一页")]/@href').extract_first()
            if next:
                next_url = response.urljoin(next)
                yield scrapy.Request(url=next_url, callback=self.parse_detail)
    ITEM_PIPELINES = {'scrapy.pipelines.images.ImagesPipeline': 1}
    
    IMAGES_URLS_FIELD = 'image_url'
    IMAGES_STORE = r'.'
  • 相关阅读:
    处理缺失值
    数据清理
    数据聚合
    ajax动态生成table
    MangeEmpHashMap
    Arraylist的雇员管理操作
    jsp获取一个对象和list对象
    Controller比较两个对象discs、outlets中的元素是否相等。相同则相应的checkbox为checked
    限制input text输入的类型(数字,字母,小数点)
    联合主键的映射运用
  • 原文地址:https://www.cnblogs.com/themost/p/7114787.html
Copyright © 2011-2022 走看看