爬虫 Scrapy框架 爬取图虫图片并下载

items.py,根据需求确定自己的数据要求

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class TodayScrapyItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass


class TuchongItem(scrapy.Item):
    title = scrapy.Field() #图片名字
    views = scrapy.Field() #浏览人数
    favorites = scrapy.Field()#点赞人数
    img_url = scrapy.Field()#图片地址

    # def get_insert_sql(self):
    #     # 存储时候用的sql语句
    #     sql = 'insert into tuchong(title,views,favorites,img_url)' 
    #           ' VALUES (%s, %s, %s, %s)'
    #     # 存储的数据
    #     data = (self['title'], self['views'], self['favorites'], self['img_url'])
    #     return (sql, data)

setting.py 设置headers和items

# -*- coding: utf-8 -*-

# Scrapy settings for today_scrapy project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'today_scrapy'

SPIDER_MODULES = ['today_scrapy.spiders']
NEWSPIDER_MODULE = 'today_scrapy.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'today_scrapy (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
  'User-Agnet':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'today_scrapy.middlewares.TodayScrapySpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'today_scrapy.middlewares.TodayScrapyDownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   # 'today_scrapy.pipelines.TodayScrapyPipeline': 300,
    'today_scrapy.pipelines.TuchongPipeline': 200,

}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

pipelines.py 将图片下载到指定文件夹

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import os
import requests

class TodayScrapyPipeline(object):
    def process_item(self, item, spider):
        return item

class TuchongPipeline(object):
    def process_item(self, item, spider):
        img_url = item['img_url'] #从items中得到图片url地址
        img_title= item['title'] #得到图片的名字
        headers = {
            'User-Agnet': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
            'cookie':'webp_enabled=1; bad_ide7dfc0b0-b3b6-11e7-b58e-df773034efe4=78baed41-a870-11e8-b7fd-370d61367b46; _ga=GA1.2.1188216139.1535263387; _gid=GA1.2.1476686092.1535263387; PHPSESSID=4k7pb6hmkml8tjsbg0knii25n6'
        }
        if not os.path.exists(img_title):
            os.mkdir(img_title)
        filename =img_url.split('/')[-1]
        with open(img_title+'/'+filename, 'wb+') as f:
            f.write(requests.get(img_url, headers=headers).content)
        f.close()
        return item

爬虫文件

tuchong.py

图片的url可以直接拼接

# -*- coding: utf-8 -*-
import scrapy
import json
from today_scrapy.items import TuchongItem


class TuchongSpider(scrapy.Spider):
    name = 'tuchong'
    allowed_domains = ['tuchong.com']
    start_urls = ['http://tuchong.com/']

    def start_requests(self):
        for pag in range(1, 20):
            referer_url = 'https://tuchong.com/rest/tags/自然/posts?page={}&count=20'.format(pag)   # url中红字部分可以换
            form_req = scrapy.Request(url=referer_url, callback=self.parse)
            form_req.headers['referer'] = referer_url
            yield form_req

    def parse(self, response):
        tuchong_info_html = json.loads(response.text)
        # print(tuchong_info_html)
        postList_c = len(tuchong_info_html['postList'])
        # print(postList_c)
        for c in range(postList_c):
            print(c)
            # print(tuchong_info_html['postList'][c])
            title = tuchong_info_html['postList'][c]['title']
            print('图集名称:'+title)
            views = tuchong_info_html['postList'][c]['views']
            print('有'+str(views)+'人浏览')
            favorites = tuchong_info_html['postList'][c]['favorites']
            print('喜欢的人数:'+str(favorites))
            images_c = len(tuchong_info_html['postList'][c]['images'])
            for img_c in range(images_c):
                user_id = tuchong_info_html['postList'][c]['images'][img_c]['user_id']
                img_id = tuchong_info_html['postList'][c]['images'][img_c]['img_id']
                img_url = 'https://photo.tuchong.com/{}/f/{}.jpg'.format(user_id,img_id)
                item = TuchongItem()
                item['title'] = title
                item['img_url'] = img_url
            # 返回我们的item
                yield item