urllib3 的使用
- 安装: pip install urllib3
urllib3 是一个基于Python3的功能强大,友好的http客户端
import urllib3
# 实例化 PoolManager 对象构造请求。这个对象处理了连接池和线程安全的所有细节,不用我们自行处理
http = urllib3.PoolManager()
# 发送请求
response = http.request('GET', 'http://httpbin.org/get', fields={'user': 'long', 'love': 'hello'})
print(response.status) # 状态码
print(response.data.decode('utf-8')) # 数据
print(response.headers) # 响应头
200
{
"args": {
"love": "hello",
"user": "long"
},
"headers": {
"Accept-Encoding": "identity",
"Host": "httpbin.org",
"X-Amzn-Trace-Id": "Root=1-5e3541ae-002a2868da392ad037911930"
},
"origin": "180.137.6.157",
"url": "http://httpbin.org/get?user=long&love=hello"
}
HTTPHeaderDict({'Date': 'Sat, 01 Feb 2020 09:15:26 GMT', 'Content-Type': 'application/json', 'Content-Length': '299', 'Connection': 'keep-alive', 'Server': 'gunicorn/19.9.0', 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Credentials': 'true'})
http://httpbin.org/get
def post_test():
url = 'http://httpbin.org/post'
http = urllib3.PoolManager()
response = http.request('POST', url, fields={'user': 'long', 'pwd': 123456})
print(response.data.decode())
post_test()
{
"args": {},
"data": "",
"files": {},
"form": {
"pwd": "123456",
"user": "long"
},
"headers": {
"Accept-Encoding": "identity",
"Content-Length": "217",
"Content-Type": "multipart/form-data; boundary=3d17043d15a2ec51d2baf65ca6cec7c2",
"Host": "httpbin.org",
"X-Amzn-Trace-Id": "Root=1-5e353979-f2194286f61c9ceae23fe342"
},
"json": null,
"origin": "180.137.6.157",
"url": "http://httpbin.org/post"
}
response.stream()
response: 响应返回的数据都是字节类型,对于大量的数据我们通过 stream
来处理更好
def big_data():
http = urllib3.PoolManager()
response = http.request('GET', 'http://httpbin.org/bytes/1024', preload_content=False)
for chunk in response.stream(32):
print(chunk)
# big_data()
也可以当做一个文件对象来处理
# 也可以当做一个文件对象来处理
def file_test():
http = urllib3.PoolManager()
response = http.request('GET', 'http://httpbin.org/bytes/1024', preload_content=False)
for line in response:
print(line)
# file_test()
使用代理
可以使用urllib3.ProxyManager 进行代理操作
proxy = urllib3.ProxyManager('http://106.85.139.161:9999')
response = proxy.request('GET', 'http://httpbin.org/ip')
# print(response.status)
# print(response.data.decode('utf-8'))
添加请求头
发送请求时传递字典形式的headers参数去指定请求头
headers = {
'user-agnet': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36'
}
http = urllib3.PoolManager()
response = http.request('GET', 'http://httpbin.org/headers', headers=headers)
print(response.data.decode())
{
"headers": {
"Accept-Encoding": "identity",
"Host": "httpbin.org",
"User-Agnet": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36",
"X-Amzn-Trace-Id": "Root=1-5e353fcc-7bd4f0203529ac98d781b4e8"
}
}