Scrapy之下载中间件中的代理中间件HttpProxyMiddleware

简单的使用内置的代理中间件

# -*- coding: utf-8 -*-
import scrapy
from wyb.items import WybItem
from scrapy.dupefilters import RFPDupeFilter
from scrapy.http.response.html import HtmlResponse
from scrapy.http.cookies import CookieJar
from urllib.parse import urlencode
from scrapy.http import Request


class ChoutiSpider(scrapy.Spider):
    name = 'chouti'
    # 爬取定向的网页 只允许这个域名的
    allowed_domains = ['chouti.com']
    start_urls = ['https://dig.chouti.com/']

    def start_requests(self):
        import os
        # 代理设置 downloadmiddleware中httppxoxy
        os.environ['HTTP_PROXY'] = '1.1.1.2'
        os.environ['HTTPS_PROXY'] = 'http://root:woshinizuzong@192.168.10.10:8888/'
        # 方式1
        for url in self.start_urls:
            yield Request(url=url)
        # 方式2
        # req_list = []
        # for url in self.start_urls:
        #     req_list.append(Request(url=url))
        # return req_list

    def parse(self, response):
        """
        第一次访问响应response
        :param response:
        :return:
        """
        page_list = response.xpath('//div[@id="dig_lcpage"]//a/@href').extract()
        for page in page_list:
            from scrapy.http import Request
            page = "https://dig.chouti.com"+page
            # 继续发请求，回调函数parse
            yield Request(url=page, callback=self.parse, meta={"proxy": "http://root:woshinizuzong@192.168.10.10:8888/"})

 源码分析

 

 

 

 

  

 

 

 

 总结：

　　如果要简单的使用只需在请求刚开始的时候定义os.environ进程环境变量

        import os
        # 代理设置 downloadmiddleware中httppxoxy
        os.environ['HTTP_PROXY'] = '1.1.1.2'
        os.environ['HTTPS_PROXY'] = 'http://root:woshinizuzong@192.168.10.10:8888/'


　　　　 yield Request(url=page, callback=self.parse, meta={"proxy": "http://root:woshinizuzong@192.168.10.10:8888/"})   # meta优先级比environ更高

 　　如果要自定义下载代理中间件需要定义这些方法

def __init__(self, auth_encoding='latin-1'):

@classmethod
def from_crawler(cls, crawler):

def _basic_auth_header(self, username, password):

def _get_proxy(self, url, orig_type):

10 def process_request(self, request, spider):

def _set_proxy(self, request, scheme): 

自定义下载代理中间件方式一

#!/usr/bin/env python
# -*- coding:utf-8 -*-
import base64
import random
from six.moves.urllib.parse import unquote, urlunparse

try:
    from urllib2 import _parse_proxy
except ImportError:
    from urllib.request import _parse_proxy
from scrapy.utils.python import to_bytes


class WybProxyMiddleware(object):

    def _basic_auth_header(self, username, password):
        user_pass = to_bytes(
            '%s:%s' % (unquote(username), unquote(password)),
            encoding='latin-1')
        return base64.b64encode(user_pass)

    def process_request(self, request, spider):
        PROXIES = [
            "http://root:woshinizuzong@192.168.10.10:8888/",
            "http://root:woshinizuzong@192.168.10.11:8888/",
            "http://root:woshinizuzong@192.168.10.12:8888/",
            "http://root:woshinizuzong@192.168.10.13:8888/",
            "http://root:woshinizuzong@192.168.10.14:8888/",
            "http://root:woshinizuzong@192.168.10.15:8888/",
        ]
        # url = "http://root:woshinizuzong@192.168.10.10:8888/"
        url = random.choice(PROXIES)
        orig_type = ""
        proxy_type, user, password, hostport = _parse_proxy(url)
        proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))

        if user:
            creds = self._basic_auth_header(user, password)
        else:
            creds = None

        request.meta['proxy'] = proxy_url
        if creds:
            request.headers['Proxy-Authorization'] = b'Basic ' + creds

自定义下载代理中间件方式二，源码看了很轻松自己定义

class ProxyMiddleware(object):
    def process_request(self, request, spider):
        PROXIES = [
            {'ip_port': '220.230.240.200:80', 'user_pass': 'woshinizuzong'},
            {'ip_port': '220.230.240.201:80', 'user_pass': 'woshinidie'},
            {'ip_port': '220.230.240.202:8888', 'user_pass': 'woshiniye'},
            {'ip_port': '220.230.240.203:80', 'user_pass': 'caonidaba'},
            {'ip_port': '220.230.240.204:80', 'user_pass': 'jiaowolaoba'},
            {'ip_port': '220.230.240.205:8888', 'user_pass': 'shuowodiaoda'},
        ]
        proxy = random.choice(PROXIES)
        if proxy['user_pass'] is not None:
            request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])
            encoded_user_pass = base64.b64decode(to_bytes(proxy['user_pass']))
            request.headers['Proxy-Authorization'] = to_bytes('Basic ' + encoded_user_pass)
        else:
            request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])

 配置文件中

 

 好奇看下Scrapy的配置文件