Python模块解析之SocketServer(二)_BaseServer基类

　　我们从上一篇文章中的类图可以看出 BaseServer 类是SocketServer模块中所有类的基类。BaseServer类规定了整个模块的框架，所以我们先仔细了解BaseServer类

我们先把整个BaseServer的源码贴出来

　　

import socket
import select
import sys
import os
import errno
try:
    import threading
except ImportError:
    import dummy_threading as threading

__all__ = ["TCPServer","UDPServer","ForkingUDPServer","ForkingTCPServer",
           "ThreadingUDPServer","ThreadingTCPServer","BaseRequestHandler",
           "StreamRequestHandler","DatagramRequestHandler",
           "ThreadingMixIn", "ForkingMixIn"]
if hasattr(socket, "AF_UNIX"):
    __all__.extend(["UnixStreamServer","UnixDatagramServer",
                    "ThreadingUnixStreamServer",
                    "ThreadingUnixDatagramServer"])

def _eintr_retry(func, *args):
    """restart a system call interrupted by EINTR"""
    while True:
        try:
            return func(*args)
        except (OSError, select.error) as e:
            if e.args[0] != errno.EINTR:
                raise

class BaseServer:

    """Base class for server classes.

    Methods for the caller:

    - __init__(server_address, RequestHandlerClass)
    - serve_forever(poll_interval=0.5)
    - shutdown()
    - handle_request()  # if you do not use serve_forever()
    - fileno() -> int   # for select()

    Methods that may be overridden:

    - server_bind()
    - server_activate()
    - get_request() -> request, client_address
    - handle_timeout()
    - verify_request(request, client_address)
    - server_close()
    - process_request(request, client_address)
    - shutdown_request(request)
    - close_request(request)
    - handle_error()

    Methods for derived classes:

    - finish_request(request, client_address)

    Class variables that may be overridden by derived classes or
    instances:

    - timeout
    - address_family
    - socket_type
    - allow_reuse_address

    Instance variables:

    - RequestHandlerClass
    - socket

    """

    timeout = None

    def __init__(self, server_address, RequestHandlerClass):
        """Constructor.  May be extended, do not override."""
        self.server_address = server_address
        self.RequestHandlerClass = RequestHandlerClass
        self.__is_shut_down = threading.Event()
        self.__shutdown_request = False

    def server_activate(self):
        """Called by constructor to activate the server.

        May be overridden.

        """
        pass

    def serve_forever(self, poll_interval=0.5):
        """Handle one request at a time until shutdown.

        Polls for shutdown every poll_interval seconds. Ignores
        self.timeout. If you need to do periodic tasks, do them in
        another thread.
        """
        self.__is_shut_down.clear()
        try:
            while not self.__shutdown_request:
                # XXX: Consider using another file descriptor or
                # connecting to the socket to wake this up instead of
                # polling. Polling reduces our responsiveness to a
                # shutdown request and wastes cpu at all other times.
                r, w, e = _eintr_retry(select.select, [self], [], [],
                                       poll_interval)
                if self in r:
                    self._handle_request_noblock()
        finally:
            self.__shutdown_request = False
            self.__is_shut_down.set()

    def shutdown(self):
        """Stops the serve_forever loop.

        Blocks until the loop has finished. This must be called while
        serve_forever() is running in another thread, or it will
        deadlock.
        """
        self.__shutdown_request = True
        self.__is_shut_down.wait()

    # The distinction between handling, getting, processing and
    # finishing a request is fairly arbitrary.  Remember:
    #

    # - handle_request() is the top-level call.  It calls
    #   select, get_request(), verify_request() and process_request()
    # - get_request() is different for stream or datagram sockets
    # - process_request() is the place that may fork a new process
    #   or create a new thread to finish the request
    # - finish_request() instantiates the request handler class;
    #   this constructor will handle the request all by itself

    def handle_request(self):
        """Handle one request, possibly blocking.

        Respects self.timeout.
        """
        # Support people who used socket.settimeout() to escape
        # handle_request before self.timeout was available.
        timeout = self.socket.gettimeout()
        if timeout is None:
            timeout = self.timeout
        elif self.timeout is not None:
            timeout = min(timeout, self.timeout)
        fd_sets = _eintr_retry(select.select, [self], [], [], timeout)
        if not fd_sets[0]:
            self.handle_timeout()
            return
        self._handle_request_noblock()

    def _handle_request_noblock(self):
        """Handle one request, without blocking.

        I assume that select.select has returned that the socket is
        readable before this function was called, so there should be
        no risk of blocking in get_request().
        """
        try:
            request, client_address = self.get_request()
        except socket.error:
            return
        if self.verify_request(request, client_address):
            try:
                self.process_request(request, client_address)
            except:
                self.handle_error(request, client_address)
                self.shutdown_request(request)

    def handle_timeout(self):
        """Called if no new request arrives within self.timeout.

        Overridden by ForkingMixIn.
        """
        pass

    def verify_request(self, request, client_address):
        """Verify the request.  May be overridden.

        Return True if we should proceed with this request.

        """
        return True

    def process_request(self, request, client_address):
        """Call finish_request.

        Overridden by ForkingMixIn and ThreadingMixIn.

        """
        self.finish_request(request, client_address)
        self.shutdown_request(request)

    def server_close(self):
        """Called to clean-up the server.

        May be overridden.

        """
        pass

    def finish_request(self, request, client_address):
        """Finish one request by instantiating RequestHandlerClass."""
        self.RequestHandlerClass(request, client_address, self)

    def shutdown_request(self, request):
        """Called to shutdown and close an individual request."""
        self.close_request(request)

    def close_request(self, request):
        """Called to clean up an individual request."""
        pass

    def handle_error(self, request, client_address):
        """Handle an error gracefully.  May be overridden.

        The default is to print a traceback and continue.

        """
        print '-'*40
        print 'Exception happened during processing of request from',
        print client_address
        import traceback
        traceback.print_exc() # XXX But this goes to stderr!
        print '-'*40

我们先从BaseServer的介绍文档来看:

Methods for the caller:

- __init__(server_address, RequestHandlerClass) 初始化,传入构建一个Socket服务器所必需的address以及每个请求到达时的处理类
- serve_forever(poll_interval=0.5) 启动服务器，服务器进入一个无限循环的状态,poll_interval用来设置循环间隔时间,默认为0.5秒。
- shutdown()  关闭服务器
- handle_request() # if you do not use serve_forever()  处理请求的方法，如果你没有调用server_forver方法，请求到达时将会调用此方法
- fileno() -> int # for select()    文件描述符,相当于一个句柄。我们知道在Unix中一切皆为文件，系统操纵每个文件时通过 文件描述符对文件来进行文件操纵的。

以上的方法都是一些基础的方法BaseServer里面已经实现,下面的一些方法可能就需要由子类来重新定义或覆盖

Methods that may be overridden:

- server_bind()  服务器绑定，被构造器调用
- server_activate() 服务器激活
- get_request() -> request, client_address 获取请求的socket
- handle_timeout()  服务器超时处理
- verify_request(request, client_address)  验证请求
- server_close()  关闭服务器
- process_request(request, client_address)  处理请求
- shutdown_request(request)  关闭请求
- close_request(request)   关闭请求
- handle_error()   错误处理

子类的方法

Methods for derived classes:

- finish_request(request, client_address)

可能被子类或者实例覆盖的的类变量

Class variables that may be overridden by derived classes or
instances:

- timeout 超时时间
- address_family  地址类型
- socket_type  socket类型 TCP/UDP
- request_queue_size (only for stream sockets)  请求队列大小,只是针对 流socket
- allow_reuse_address  是否允许重置监听的地址

Instance variables:  实例变量

- server_address  服务器地址(服务监听的地址)
- RequestHandlerClass  请求处理类
- socket   套接字

"""

从上面BaseSever的介绍中我们已经大致了解了整个SocketSever的设计思路。构建一个SocketServer,定义SocketServer的一些基本操作 启动,关闭,处理请求，超时处理，错误处理。定义一个SocketServer所需要信息和步骤都一样所以它把他放在初始化构造器里面，由我们在创建BaseServer时传入。启动，关闭这些基础功能在BaseServer里面已经实现。针对我们最感兴趣的 "处理请求" 它把请求处理 抽离出来成一个类，让我们自己编写处理类，因为如何处理只有我们自己知道。把变化抽离出来,，这样提高了服务器可扩展功能。针对不同Socket的更详细的方法 当然就由不同的SocketServer子类来重新定义喽。

有了上面的大致认识后，在研究BaseServer的源代码就有一个主体方向了：

timeout = None

定义一个 超时 类变量

def __init__(self, server_address, RequestHandlerClass):
        """Constructor.  May be extended, do not override."""
        self.server_address = server_address
        self.RequestHandlerClass = RequestHandlerClass
        self.__is_shut_down = threading.Event()
        self.__shutdown_request = False

初始化构造器,定义了 构造了一个SocketServer 所必须的最基本的server_address 以及我们自己编写的RequestHandlerClass类。我们看到一个 __is_shut_down私有成员变量 被赋值了一个 threading.Event()可以看出这个变量是用来控制 服务器关闭的。还有一个__shutdown_request标识是否关闭请求。

    def server_activate(self):
        """Called by constructor to activate the server.

        May be overridden.

        """
        pass

服务器激活 由子类覆盖实现。

    def serve_forever(self, poll_interval=0.5):
        """Handle one request at a time until shutdown.

        Polls for shutdown every poll_interval seconds. Ignores
        self.timeout. If you need to do periodic tasks, do them in
        another thread.
        """
        self.__is_shut_down.clear()
        try:
            while not self.__shutdown_request:
                # XXX: Consider using another file descriptor or
                # connecting to the socket to wake this up instead of
                # polling. Polling reduces our responsiveness to a
                # shutdown request and wastes cpu at all other times.
                r, w, e = _eintr_retry(select.select, [self], [], [],
                                       poll_interval)
                if self in r:
                    self._handle_request_noblock()
        finally:
            self.__shutdown_request = False
            self.__is_shut_down.set()

从这段代码中我们可以看出BaseServer是一个非阻塞的SocketServer.用Threading.Event来控制服务器的关闭，那么self.__is_shutdown是如何来控制服务器的关闭呢？

我们接着看:

    def shutdown(self):
        """Stops the serve_forever loop.

        Blocks until the loop has finished. This must be called while
        serve_forever() is running in another thread, or it will
        deadlock.
        """
        self.__shutdown_request = True
        self.__is_shut_down.wait()

从shutdown()方法中我们 通过调用self.__shutdown_request来停止 serve_forever的循环,但是通过调用self.__is_shut_down.wait()方法是它一直处于阻塞状态，知道调用self.__is_shut_down.set()后进程才算结束。也就是说知道处理完所有请求才关闭掉进程。如果server_forver()方法运行在另一个线程中这必须调用该方法否则会出现死锁。

至于用select实现非阻塞的处理.关于threading.Event 和 select.select可以看我之前的专门介绍Event和select的文章。可能现在有点搞不懂，因为这要结合后面的代码 然后串起来 才能够明白。这里我们只需要明白是通过select来实现非阻塞的服务的，具体细节后面会结合代码来讨论。

下面要介绍的两个方法至关重要，我们要结合 server_forver() 来 把整个服务器如何通过非阻塞方式来  接收请求 处理请求  这个最重要的处理步骤弄明白，

我们先来看看 handler_request()的介绍:

# The distinction between handling, getting, processing and
# finishing a request is fairly arbitrary. Remember:
#

一个请求的handling, getting, processing, finishing它们之间的区别是非常重要的。记住: 

# - handle_request() is the top-level call. It calls
# select, get_request(), verify_request() and process_request()
# - get_request() is different for stream or datagram sockets
# - process_request() is the place that may fork a new process
# or create a new thread to finish the request
# - finish_request() instantiates the request handler class;
# this constructor will handle the request all by itself

handler_request()方法是最上层的调用.它调用了 select, get_Request(), verify_request() 和 process_request()方法.

get_request()对于 流套接字(面向连接)和 数据包套接字(面向非连接)来说 是不同的.在Python socket编程中对于  stream(流套接字)接收请求调用socket.accept(),对于 datagram(数据包套接字)接收请求时调用socket.recvfrom().所以说这个方法由子类来实现。针对不同的套接字调用不同的方法。

process_request()处理一个请求时可能会创建一个进程或者线程去处理。

finish_request()完成请求 不同的实例所做的处理也不同。这个无关紧要。

    def handle_request(self):
        """Handle one request, possibly blocking.

        Respects self.timeout.
        """
        # Support people who used socket.settimeout() to escape
        # handle_request before self.timeout was available.
        timeout = self.socket.gettimeout()
        if timeout is None:
            timeout = self.timeout
        elif self.timeout is not None:
            timeout = min(timeout, self.timeout)
        fd_sets = _eintr_retry(select.select, [self], [], [], timeout)
        if not fd_sets[0]:
            self.handle_timeout()
            return
        self._handle_request_noblock()

支持请求超时，如果请求超时(在timeout时间内没有请求到达)则调用超时处理方法 hanler_timeout()，否则调用self._handle_request_noblock()

    def _handle_request_noblock(self):
        """Handle one request, without blocking.

        I assume that select.select has returned that the socket is
        readable before this function was called, so there should be
        no risk of blocking in get_request().
        """
        try:
            request, client_address = self.get_request()
        except socket.error:
            return
        if self.verify_request(request, client_address):
            try:
                self.process_request(request, client_address)
            except:
                self.handle_error(request, client_address)
                self.shutdown_request(request)

一个非阻塞的处理请求的方法。我们在回到  server_forever()方法中：

self._handle_request_noblock()

我想可能 下面这段代码 会更明白一些。

import socket
import traceback
import select

EOL1 = b'\n\n'
EOL2 = b'\n\r\n'

socketmap = {}
r, w, e = [], [], []

response = b'HTTP/1.0 200 OK\r\nDate: Mon, 1 Jan 1996 01:01:01 GMT\r\n'
response += b'Content-Type: text/plain\r\nContent-Length: 13\r\n\r\n'
response += b'Hello, world!'

serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
serversocket.bind(('0.0.0.0', 23456))
serversocket.listen(1)
# serversocket.setblocking(0)

listening_fileno = serversocket.fileno()
socketmap[listening_fileno] = serversocket
print 'listening_fileno', listening_fileno

try:
    while True:
        r, w, e = [], [], []
        for fd in socketmap:
            r.append(fd)
            w.append(fd)
            e.append(fd)
        r, w, e = select.select(r, w, e, 1)
        for fd in r:
            request = b''
            isocket = socketmap[fd]
            if fd == listening_fileno:
                print 'accepting'
                clientsock, clientaddr = isocket.accept()
                # clientsock.setblocking(0)
                cli_fileno = clientsock.fileno()
                r.append(cli_fileno)
                w.append(cli_fileno)
                e.append(cli_fileno)
                socketmap[cli_fileno] = clientsock
            else:
                print 'reading'
                while EOL1 not in request and EOL2 not in request:
                    request += isocket.recv(1024)
                print(request.decode())

        for fd in w:
            print 'writing'
            osocket = socketmap[fd]
            osocket.send(response)
            
        for fd in e:
            esocket = socketmap[fd]
            print 'socket close', fd
            esocket.close()
            del socketmap[fd]

        print "no data coming"

except Exception, e:
    print traceback.print_exc()
    serversocket.close()

我们利用了select实现了非阻塞socket服务.

至于 后面的都是一些细节处理

    def handle_timeout(self):
        """Called if no new request arrives within self.timeout.

        Overridden by ForkingMixIn.
        """
        pass

    def verify_request(self, request, client_address):
        """Verify the request.  May be overridden.

        Return True if we should proceed with this request.

        """
        return True

    def process_request(self, request, client_address):
        """Call finish_request.

        Overridden by ForkingMixIn and ThreadingMixIn.

        """
        self.finish_request(request, client_address)
        self.shutdown_request(request)

    def server_close(self):
        """Called to clean-up the server.

        May be overridden.

        """
        pass

    def finish_request(self, request, client_address):
        """Finish one request by instantiating RequestHandlerClass."""
        self.RequestHandlerClass(request, client_address, self)

    def shutdown_request(self, request):
        """Called to shutdown and close an individual request."""
        self.close_request(request)

    def close_request(self, request):
        """Called to clean up an individual request."""
        pass

    def handle_error(self, request, client_address):
        """Handle an error gracefully.  May be overridden.

        The default is to print a traceback and continue.

        """
        print '-'*40
        print 'Exception happened during processing of request from',
        print client_address
        import traceback
        traceback.print_exc() # XXX But this goes to stderr!
        print '-'*40

总结: 其实BaseServer只是大致搭了一个 模型 。控制整体的服务流程 启动服务->接收请求->处理请求 ，对于接收和处理 则 利用 select 实现非阻塞。把请求处理抽离出来，让我们自己实现。当然对于处理 请求 process_request() 它还提供了 让我们自己实现，使我们可以利用 进程和线程来处理 实现更高效。其实只要理解了 select 和上面我贴出的那段代码 BaseServer也就理解啦。后面我们会详细的介绍处理各个子类的实现细节。