Downloader Middleware

Downloader Middleware

源码解析：

# 文件：E:MinicondaLibsite-packagesscrapycoredownloadermiddleware.py
"""
Downloader Middleware manager

See documentation in docs/topics/downloader-middleware.rst
"""
import six

from twisted.internet import defer

from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.conf import build_component_list


class DownloaderMiddlewareManager(MiddlewareManager):

    component_name = 'downloader middleware'

    @classmethod
    def _get_mwlist_from_settings(cls, settings):
        # 从settings.py或这custom_setting中拿到自定义的Middleware中间件
        '''
        'DOWNLOADER_MIDDLEWARES': {
            'mySpider.middlewares.ProxiesMiddleware': 400,
            # SeleniumMiddleware
            'mySpider.middlewares.SeleniumMiddleware': 543,
            'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
        },
        '''
        return build_component_list(
            settings.getwithbase('DOWNLOADER_MIDDLEWARES'))

    # 将所有自定义Middleware中间件的处理函数添加到对应的methods列表中
    def _add_middleware(self, mw):
        if hasattr(mw, 'process_request'):
            self.methods['process_request'].append(mw.process_request)
        if hasattr(mw, 'process_response'):
            self.methods['process_response'].insert(0, mw.process_response)
        if hasattr(mw, 'process_exception'):
            self.methods['process_exception'].insert(0, mw.process_exception)

    # 整个下载流程
    def download(self, download_func, request, spider):
        @defer.inlineCallbacks
        def process_request(request):
            # 处理request请求，依次经过各个自定义Middleware中间件的process_request方法，前面有加入到list中
            for method in self.methods['process_request']:
                response = yield method(request=request, spider=spider)
                assert response is None or isinstance(response, (Response, Request)), 
                        'Middleware %s.process_request must return None, Response or Request, got %s' % 
                        (six.get_method_self(method).__class__.__name__, response.__class__.__name__)
                # 这是关键地方
                # 如果在某个Middleware中间件的process_request中处理完之后，生成了一个response对象
                # 那么会直接将这个response return 出去，跳出循环，不再处理其他的process_request
                # 之前我们的header，proxy中间件，都只是加个user-agent，加个proxy，并不做任何return值
                # 还需要注意一点:就是这个return的必须是Response对象
                # 后面我们构造的HtmlResponse正是Response的子类对象
                if response:
                    defer.returnValue(response)
            # 如果在上面的所有process_request中，都没有返回任何Response对象的话
            # 最后，会将这个加工过的Request送往download_func，进行下载，返回的就是一个Response对象
            # 然后依次经过各个Middleware中间件的process_response方法进行加工，如下
            defer.returnValue((yield download_func(request=request,spider=spider)))

        @defer.inlineCallbacks
        def process_response(response):
            assert response is not None, 'Received None in process_response'
            if isinstance(response, Request):
                defer.returnValue(response)

            for method in self.methods['process_response']:
                response = yield method(request=request, response=response,
                                        spider=spider)
                assert isinstance(response, (Response, Request)), 
                    'Middleware %s.process_response must return Response or Request, got %s' % 
                    (six.get_method_self(method).__class__.__name__, type(response))
                if isinstance(response, Request):
                    defer.returnValue(response)
            defer.returnValue(response)

        @defer.inlineCallbacks
        def process_exception(_failure):
            exception = _failure.value
            for method in self.methods['process_exception']:
                response = yield method(request=request, exception=exception,
                                        spider=spider)
                assert response is None or isinstance(response, (Response, Request)), 
                    'Middleware %s.process_exception must return None, Response or Request, got %s' % 
                    (six.get_method_self(method).__class__.__name__, type(response))
                if response:
                    defer.returnValue(response)
            defer.returnValue(_failure)

        deferred = mustbe_deferred(process_request, request)
        deferred.addErrback(process_exception)
        deferred.addCallback(process_response)
        return deferred