在生产环境下使用生成器像服务器传输大文件

废话不多说，直接上代码

# -*- coding: utf-8 -*-
# Created by richie at 2018/10/23
import os
import sys

_ver = sys.version_info

#: Python 2.x?
is_py2 = (_ver[0] == 2)

#: Python 3.x?
is_py3 = (_ver[0] == 3)

import httplib
import string
import random
from array import array
from urllib3.exceptions import LocationParseError
from urllib3.util import parse_url

if is_py2:
    from collections import Mapping

    range = xrange
elif is_py3:
    from collections.abc import Mapping

    range = range

_BOUNDARY_CHARS = string.digits + string.ascii_letters


def to_key_val_list(value):
    if value is None:
        return None

    if isinstance(value, (str, bytes, bool, int)):
        raise ValueError('cannot encode objects that are not 2-tuples')

    if isinstance(value, Mapping):
        value = value.items()

    return list(value)


class HTTPConnection(httplib.HTTPConnection):
    """
    改写send方法  使之能够接受生成器
    """
    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

    def send(self, data):
        """Send `data' to the server."""
        if self.sock is None:
            if self.auto_open:
                self.connect()
            else:
                raise httplib.NotConnected()

        if self.debuglevel > 0:
            print "send:", repr(data)
        blocksize = 8192
        if hasattr(data, 'next'):
            for value in data:
                self.sock.sendall(value)

        elif hasattr(data, 'read') and not isinstance(data, array):
            if self.debuglevel > 0: print "sendIng a read()able"
            datablock = data.read(blocksize)
            while datablock:
                self.sock.sendall(datablock)
                datablock = data.read(blocksize)
        else:
            self.sock.sendall(data)


class Request(object):

    def __init__(self, method=None, url=None, data=None, files=None, ):
        # Default empty dicts for dict params.
        data = [] if data is None else data
        files = [] if files is None else files

        self.method = method
        self.url = url
        self.files = files
        self.data = data
        self.headers = {
            'Accept': '*/*',
            'Connection': 'keep-alive',
        }
        self.__boundary = None
        self.__item_header = []
        self.end_tag = None

        self.init_body()

    def __repr__(self):
        return '<Request [%s]>' % (self.method)

    def get_boundary(self):
        if self.__boundary is None:
            self.__boundary = ''.join(random.choice(_BOUNDARY_CHARS) for _ in range(32))
        return self.__boundary

    def init_body(self):
        """
        组装数据
        :return:
        """
        def escape_quote(s):
            return s.replace('"', '\"')

        fields = to_key_val_list(self.data or {})
        files = to_key_val_list(self.files or {})
        lines = []
        fpaths = []
        fheads = []
        if fields:
            for name, value in fields:
                lines.extend((
                    '--{0}'.format(self.get_boundary()),
                    'Content-Disposition: form-data; name="{0}"'.format(escape_quote(name)),
                    '',
                    str(value),
                ))
        if files:
            __temp = []
            for name, value in files:
                filename = os.path.basename(value)
                filename = filename.encode('utf-8')
                __temp.extend((
                    '--{0}'.format(self.get_boundary()),
                    'Content-Disposition: form-data; name="{0}"; filename="{1}"'.format(
                        escape_quote(name), escape_quote(filename)),
                    '
',
                ))
                fpaths.append(value)
                fheads.append('
'.join(__temp))
        fields_heads = '
'.join(lines) + '
'
        self.end_tag = "--" + self.get_boundary() + "--
"

        content_type = str('multipart/form-data; boundary=%s' % self.get_boundary())
        if content_type and ('content-type' not in self.headers):
            self.headers['Content-Type'] = content_type

        self.prepare_content_length(fields_heads, fheads, fpaths)
        self.data = self.prepare_data(fields_heads, fheads, fpaths)

    def prepare_data(self, fields_heads, fheads, fpaths):
        """
        通过生成器发送数据
        :param fields_heads:
        :param fheads:
        :param fpaths:
        :return:
        """
        b_size = 4096
        yield fields_heads
        for i in range(len(fheads)):
            yield fheads[i]
            file_obj = open(fpaths[i], 'rb')
            while True:
                block = file_obj.read(b_size)
                if block:
                    yield block
                else:
                    yield '
'
                    break
            yield self.end_tag

    def prepare_content_length(self, fields_heads, fheads=None, fpaths=None):
        """
        准备发送内容的大小
        :param fields_heads:
        :param fheads:
        :param fpaths:
        :return:
        """
        if fields_heads is not None:
            length = len(fields_heads)
            if fpaths:
                for fpath in fpaths:
                    length += os.path.getsize(fpath) + len('
')  # file length
                for fhead in fheads:
                    length += len(fhead)
                length += len(self.end_tag)
            if length:
                self.headers['Content-Length'] = str(length)

        elif self.method not in ('GET', 'HEAD') and self.headers.get('Content-Length') is None:
            self.headers['Content-Length'] = '0'

    def send(self):
        """
        通过httplib发送请求
        :return:
        """
        # Support for unicode domain names and paths.
        try:
            scheme, auth, host, port, path, query, fragment = parse_url(self.url)
        except LocationParseError as e:
            raise Exception(*e.args)

        with HTTPConnection(host, port=port) as conn:
            conn.request(self.method.upper(), self.url, self.data, self.headers)
            # Receive the response from the server
            try:
                # For Python 2.7, use buffering of HTTP responses
                response = conn.getresponse(buffering=True)
            except TypeError:
                # For compatibility with Python 3.3+
                response = conn.getresponse()
            status = response.status
            content = response.read()
        return status, content