from gevent import monkey
from gevent.pool import Pool
# 猴子补丁,替换底层thread/socket实现非阻塞
monkey.patch_all()
import gevent
import requests
def func(url):
print('GET: %s' % url)
resp = requests.get(url)
data = resp.text
print('%d bytes received from %s.' % (len(data), url))
result_list.append(data)
result_list = []
if __name__ == "__main__":
# gevent实现协程
gevent.joinall([
gevent.spawn(func, 'https://www.python.org/'),
gevent.spawn(func, 'https://www.yahoo.com/'),
gevent.spawn(func, 'https://github.com/'),
])
print(f'result length:{len(result_list)}')
# gevent限制协程数量
inputs = ['https://www.python.org/', 'https://www.yahoo.com/', 'https://github.com/', 'https://www.baidu.com',
'https://www.qq.com', 'https://www.meituan.com', 'https://www.iqiyi.com']
pool = Pool(5)
threads = [pool.spawn(func, i) for i in inputs]
gevent.joinall(threads)
print(f'finish1, data length:{len(result_list)}')
结果:
"D:Program FilesPython36python.exe" "D:Program FilesJetBrainsPyCharm Community Edition 2019.2.4helperspydevpydevd.py" --multiproc --qt-support=auto --client 127.0.0.1 --port 62826 --file D:/04_project/tools/tools/yield_request.py
pydev debugger: process 3876 is connecting
Connected to pydev debugger (build 192.7142.42)
GET: https://www.python.org/
GET: https://www.yahoo.com/
GET: https://github.com/
50225 bytes received from https://www.python.org/.
219487 bytes received from https://github.com/.
632593 bytes received from https://www.yahoo.com/.
result length:3
GET: https://www.python.org/
GET: https://www.yahoo.com/
GET: https://github.com/
GET: https://www.baidu.com
GET: https://www.qq.com # 执行5个协程后等待返回结果
2443 bytes received from https://www.baidu.com.
GET: https://www.meituan.com
99681 bytes received from https://www.qq.com.
GET: https://www.iqiyi.com
50225 bytes received from https://www.python.org/.
38273 bytes received from https://www.meituan.com.
219493 bytes received from https://github.com/.
325656 bytes received from https://www.iqiyi.com.
628076 bytes received from https://www.yahoo.com/.
finish1, data length:10
Process finished with exit code 0