性能

。。。

“”“
可以实现并发,请求发送出去后和返回之前,中间时期线程空闲
“”“



#  编写方式一： 直接返回处理

from concurrent.futures import ThreadPoolExecutor
import requests


def task(url):
    response = requests.get(url=url)
    print(url,response)

pool = ThreadPoolExecutor(5)

url_list = [
    'http://www.cnblogs.com/wupeiqi',
    'http://www.huaban.com/favorite/beauty/',
    'http://www.bing.com',
    'http://www.zhihu.com',
    'http://www.sina.com',
    'http://www.baidu.com',
    'http://www.autohome.com.cn',
]

for url in url_list:
    pool.submit(task,url)

pool.shutdown(wait=True)

---------------------------------------------------
# 编写方式二: 通过回调函数处理

from concurrent.futures import ThreadPoolExecutor
import requests


def task(url):
    response = requests.get(url=url)
    return response

def done(future,*args,**kwargs):
    response = future.result()
    print(response.url,response)


pool = ThreadPoolExecutor(5)

url_list = [
    'http://www.cnblogs.com/wupeiqi',
    'http://www.huaban.com/favorite/beauty/',
    'http://www.bing.com',
    'http://www.zhihu.com',
    'http://www.sina.com',
    'http://www.baidu.com',
    'http://www.autohome.com.cn',
]

for url in url_list:
    v = pool.submit(task,url)
    v.add_done_callback(done)


pool.shutdown(wait=True)

协程 异步IO
	- asyncio
		内部实现了异步IO操作,但没有实现全部请求,没有实现Http请求
		- 示例1: asyncio.sleep(5)
		- 示例2: 自己封装Http数据包
		- 示例3: asyncio + aiohttp
			- aiohttp模块: 封装Http数据包 pip3 install aiohttp
		- 示例4: asyncio+requests
			- request模块: 封装Http数据包 pip3 install request
    - gevent，greenlet+异步IO
            pip3 install greenlet
            pip3 install gevent
        - 示例1：gevent+requests
        - 示例2：gevent（协程池，最多发多少个请求）+requests
        - 示例3：gevent+requests => grequests
        	pip3 install grequests
    - Twisted
        pip3 install twisted
    - Tornado
        pip3 install tornado
    
    =====> gevent > Twisted > Tornado > asyncio

　　

协程 异步IO
    - asyncio
        内部实现了异步IO操作,但没有实现全部请求,没有实现Http请求
        - 示例1: asyncio.sleep(5)
        - 示例2: 自己封装Http数据包
        - 示例3: asyncio + aiohttp
            - aiohttp模块: 封装Http数据包 pip3 install aiohttp
        - 示例4: asyncio+requests
            - request模块: 封装Http数据包 pip3 install request
    - gevent，greenlet+异步IO
            pip3 install greenlet
            pip3 install gevent
        - 示例1：gevent+requests
        - 示例2：gevent（协程池，最多发多少个请求）+requests
        - 示例3：gevent+requests => grequests
            pip3 install grequests
    - Twisted
        pip3 install twisted
    - Tornado
        pip3 install tornado
    
    =====> gevent > Twisted > Tornado > asyncio

#!/usr/bin/env python
# -*- coding:utf-8 -*-
from twisted.internet import defer
from twisted.web.client import getPage
from twisted.internet import reactor

def one_done(arg):
    print(arg)

def all_done(arg):
    print('done')
    reactor.stop()

@defer.inlineCallbacks
def task(url):
    res = getPage(bytes(url, encoding='utf8')) # 发送Http请求
    res.addCallback(one_done)
    yield res

url_list = [
    'http://www.cnblogs.com',
    'http://www.cnblogs.com',
    'http://www.cnblogs.com',
    'http://www.cnblogs.com',
]

defer_list = [] # [特殊，特殊，特殊(已经向url发送请求)]
for url in url_list:
    v = task(url)
    defer_list.append(v)

d = defer.DeferredList(defer_list)
d.addBoth(all_done)


reactor.run() # 死循环

#IO多路复用:  监听多个socket对象
#异步IO:    非阻塞的socket + IO多路复用

　　

import socket
import select

############### HTTP请求本质 阻塞

"""
sk = socket.socket()
# 连接
sk.connect(('www.baidu.com',80))    # IO阻塞
print('连接成功')

# 发送消息
sk.send(b'GET / HTTP/1.0
Host:www.baidu.com

')

# 等待服务端响应
data = sk.recv(8096)    # IO阻塞
print(data)

#关闭连接
sk.close()
"""

############### HTTP请求本质 非阻塞


# sk = socket.socket()
# sk.setblocking(False)
#
# try:
#     # 连接
#     sk.connect(('www.baidu.com',80))    # IO阻塞
#     print('连接成功')
# except BlockingIOError as e:
#     print(e)
#
# # 发送消息
# sk.send(b'GET / HTTP/1.0
Host:www.baidu.com

')
#
# # 等待服务端响应
# data = sk.recv(8096)    # IO阻塞
# print(data)
#
# #关闭连接
# sk.close()

#########################


class HttpRequest:
    def __init__(self,sk,host,callback):
        self.socket = sk
        self.host = host
        self.callback = callback
    def fileno(self):
        return self.socket.fileno()




class AsyncRequest:
    def __init__(self):
        self.conn = []
        self.connection = []    #用于检测是否监听成功

    def add_request(self,host,callback):
        try:
            sk = socket.socket()
            sk.setblocking(0)
            sk.connect((host,80))
        except BlockingIOError as e:
            pass
        request = HttpRequest(sk,host,callback)
        self.conn.append(request)
        self.connection.append(request)

    def run(self):
        while True:
            rlist,wlist,elist = select.select(self.conn,self.connection,self.conn,0.05)
            for w in wlist:
                print(w.host,'连接成功...')
                # 只要能循环到,表示socket和服务端已经连接成功
                tpl = "GET / HTTP/1.0
Host:%s

" % (w.host,)
                w.socket.send(bytes(tpl,encoding='utf-8'))
                self.connection.remove(w)
            for r in rlist:

                # r 是Httprequest
                recv_data = bytes()
                while True:
                    try:
                        chunck = r.socket.recv(8096)
                        recv_data += chunck
                    except Exception as e:
                        break
                # print(r.host,'有数据返回',recv_data)
                r.callback(recv_data)
                r.socket.close()
                self.conn.remove(r)

            if len(self.conn) == 0:
                break

def f1(data):
    print('保存到文件',data)

def f2(data):
    print('保存到数据库',data)


url_list = [
    {'host':'www.baidu.com','callback':f1},
    {'host':'www.cnblogs.com','callback':f2},
    {'host':'cn.bing.com','callback':f2}
]

req = AsyncRequest()
for item in url_list:
    req.add_request(item["host"],item["callback"])
req.run()

沛齐