scrapy

1. 创建项目

scrapy startproject <projectName>

2. 创建爬虫(项目目录下  cd <projectName>)

scrapy genspider <spiderName> <start_url>

3. 运行爬虫(项目目录下)

scrapy crawl <spiderName>

======

爬虫技巧

设置setting.py

1. 设置不遵循 ROBOTSTXT_OBEY

ROBOTSTXT_OBEY = False

2. 设置延时

DOWNLOAD_DELAY = 3

3. 设置  USER_AGENT  和  DEFAULT_REQUEST_HEADERS

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0;WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
   'Accept-Language': 'en',
   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0;WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}

4. 设置 中间件（设置爬虫的headers和proxoy）

　　4.1 下载中间件 设置开启

// project_dir/settings.py
# Enable or disable downloader middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
    'ADemo.middlewares.ProxyMilldeware': 301,    # 下载中间件 [调度器与引擎之间的数据管道]
}

 　　

 // project_dir/middlewares.py

 1 class ProxyMilldeware(object):   # request 来自调度器 [用去处理或配置到下载器之前的 request 对象]
    def process_request(self, request, spider):
        # print('*'*100)
        request.meta['proxy'] = 'http://127.0.0.1:1080'
        request.headers.setdefault('User-Agent', '在这里设置成你的浏览器用户代理')
           request.cookies = {'在这里设置你的cookies， 以字典的格式'}

    def process_response(self, request, response, spider):  # response 来自下载器 [用于处理下载器下载过来的 response 对象]
        print(response.status)
        print(response.text)
        return response

　　4.2 爬虫中间件 设置开启

// project_dir/settings.py
1 # Enable or disable spider middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
SPIDER_MIDDLEWARES = {5     'ADemo.middlewares.SpiderMiddleware': 543, # 爬虫中间件  [spider与引擎之间的数据管道]
}

 5. 设置是否允许重定向，重试次数

// project_dir/settings.py

1 # 设置重试次数
RETRY_ENABLED = False
# RETRY_TIMES = 1
# RETRY_HTTP_CODECS = [400, 500]

# 设置重定向
REDIRECT_ENABLED = False

 6. 日志

{'downloader/request_bytes': 288,   　　   　　　　　　　　　　　　　　　　　 # 请求字节数
 'downloader/request_count': 1,　　　　　　　　　　　　　　　　　　　　　　　　 # 请求次数
 'downloader/request_method_count/GET': 1,　　　　　　　　　　　　　　　　　  # get 请求数
 'downloader/response_bytes': 71860,　　　　　　　　　　　　　　　　　　　　　 # 响应字节数
 'downloader/response_count': 1,　　　　　　　　　　　　　　　　　　　　　　　　# 响应次数
 'downloader/response_status_count/200': 1,　　　　　　　　　　　　　　　　　 # 响应成功状态码为200的次数
 'finish_reason': 'finished',　　　　　　　　　　　　　　　　　　　　　　　　　　# 结束原因： 运行完成！
 'finish_time': datetime.datetime(2018, 1, 10, 6, 35, 53, 633178),      # 程序结束时间
 'log_count/DEBUG': 2,　　　　　　　　　　　　　　　　　　
 'log_count/INFO': 7,
 'response_received_count': 1,
 'scheduler/dequeued': 1,
 'scheduler/dequeued/memory': 1,
 'scheduler/enqueued': 1,
 'scheduler/enqueued/memory': 1,
 'start_time': datetime.datetime(2018, 1, 10, 6, 35, 51, 585036)}

 ======      settings.py 示例文件      ======

# -*- coding: utf-8 -*-

# Scrapy settings for ADemo project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'ADemo'

SPIDER_MODULES = ['ADemo.spiders']
NEWSPIDER_MODULE = 'ADemo.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0;WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False  # 一般设为False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False


# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
   'Accept-Language': 'en',
   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0;WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}

# Enable or disable spider middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
SPIDER_MIDDLEWARES = {
    # 'ADemo.middlewares.AdemoSpiderMiddleware': 543,
    # 'ADemo.middlewares.SpiderMiddleware': 543,
}

# Enable or disable downloader middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
    'ADemo.middlewares.ProxyMilldeware': 301,
}

# Enable or disable extensions
# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
#    'ADemo.pipelines.AdemoPipeline': 300,
#}

# Enable and configure the AutoThrottle extension (disabled by default)
# See http://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

# 设置重试次数
RETRY_ENABLED = True
RETRY_TIMES = 2
# RETRY_HTTP_CODECS = [320, 500]

# 设置重定向
# REDIRECT_ENABLED = False


// 分布式爬虫，配置 redis 

# SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# REDIS_URL = "redis://user:password@host:port"

 cookies的处理 [将前端浏览器request请求头中的headers字符串中的cookies字符串复制给cookies]

cookies_string = '__utma=160444991.1773391101.1513841173.1515472372.1515564585.4; __utmc=160444991; __utmz=160444991.1515564585.4.4.utmcsr=baidu|utmccn=(organic)|utmcmd=organic'
cookies = {}
for item in cookies_string.split(';'):
    key = item.split('=')[0].strip()
    value = item.split('=')[1].strip()
    cookies[key] = value
7 print(cookies)

headers的处理 [将前端浏览器request请求头中的headers字符串复制给headers_string]

headers_string = ''
headers = {}
for item in headers_string.split('
'):
    lst = item.split(':')
    key = lst[0].strip()
    value = lst[1]
    headers[key] = value.strip()
print(headers)