基本库requestes

requests, 轻松实现cookies,登陆验证,代理设置

import requests
r = requests.get('http://www.baidu.com')
print(r.text)

可以通过params这个参数进行附加的额外信息

data = {"name":"slz",
             "age":"19",
}
r=requests.get("http://httpbin/get",data=data)

网页返回的信息,type(r.text)是JSON格式的str类型,调用json(),str可以转化为dict

type(r.text)        #<class 'str'>
type(r.json())        #<class 'dict'>

如果结果不是JSON,便会出现解析错误

抓取二进制数据

图片,音频,视频这些文件本质上都是由二进制码组成,所以,想要抓取他们,就要拿到他们的二进制码

#抓取的内容是站点图标
r = requests.get('https://github.com/favicon.ico')
print(r.text)    #:�������OL������������������!������4@���8���

print(r.content)    #b'x00x00x01x00x02x00x10x10x00x00x01x00

前者出现了乱码,由于图片是二进制数据,所以前者在打印时转化为str类型,也就是图片直接转化成字符串,理所当然的出现乱码

后者的结果前带有一个b,代表着bytes类型的数据

接着,我们将提取到的图片保存下来

with open('favicon.ico','wb') as f: 
    f.write(r.content)

添加headers传递头信息

>>> headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'}

r = requests.get('http://www.baidu.com',headers=headers)

User-Agent通过浏览器Network -- all获取

响应

r.status_code   #200
r.headers        #{'Bdpagetype': '1', 'Bdqid': '0xebcc91ce0003f2b0', 'Cache-Control': 'private', 'Connection': 'Keep-Alive', 'Content-Encoding': 'gzip', 'Content-Type': 'text/html', 'Cxy_all': 'baidu+053736d121f7999293df1fcf6794b680', 'Date': 'Tue, 05 Jun 2018 11:36:31 GMT', 'Expires': 'Tue, 05 Jun 2018 11:36:04 GMT', 'Server': 'BWS/1.1', 'Set-Cookie': 'BDSVRTM=0; path=/, BD_HOME=0; path=/, H_PS_PSSID=26599_1466_26458_21087; path=/; domain=.baidu.com', 'Strict-Transport-Security': 'max-age=172800', 'Vary': 'Accept-Encoding', 'X-Ua-Compatible': 'IE=Edge,chrome=1', 'Transfer-Encoding': 'chunked'}

r.cookies    #<RequestsCookieJar[Cookie(version=0, name='H_PS_PSSID', value='26599_1466_26458_21087', port=None, port_specified=False, domain='.baidu.com', domain_specified=True, domain_initial_dot=True, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={}, rfc2109=False), Cookie(version=0, name='BDSVRTM', value='0', port=None, port_specified=False, domain='www.baidu.com', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={}, rfc2109=False), Cookie(version=0, name='BD_HOME', value='0', port=None, port_specified=False, domain='www.baidu.com', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={}, rfc2109=False)]>

r.url    #'https://www.baidu.com/'

r.history    #[<Response [302]>] (请求历史)

文件上传

files = {'file':open('favicon.ico','rb')}
r = requests.post("http://httpbin.org/post",files=files)

cookies(我们可以理解为里面有登陆的凭证)

以知乎为例,先用浏览器登陆,在浏览器中查看cookie

headers{
'cookie':'d_c0="AACk84M3qQ2PTtPu6KwDs8IjEmXXzM7Ksas=|1527474250"; q_c1=36ca9c436e5a401e81dd6b54efc104c5|1527474250000|1527474250000; _zap=8dd8419d-b790-47ef-8897-d156601415e3; l_cap_id="YjY4ZmRhYTc0NTFjNDNiZjg3YzU1NGM3ZDZhMWVjNTM=|1527994443|35ab43ae969180ea6abf785210c5a7561c7a470f"; r_cap_id="NGY0Y2U3MWIwNDBiNDY5NDlhMGJhMzYzYzdlNzRiOTc=|1527994443|da38fc2a73bd1d72e0bd529fbda3dac2b2441971"; cap_id="MWU4YTk1MGVlODE0NGEzMzk4NjVjZmNjODQ5MTkwODk=|1527994443|a9eb9c176fcfff78e1d143df5e3f24cb26cbbba7"; tgw_l7_route=170010e948f1b2a2d4c7f3737c85e98c; _xsrf=8dd3bd87-edb1-4cea-894f-362cbf2e507c; z_c0="2|1:0|10:1528199820|4:z_c0|92:Mi4xSUxsZkJBQUFBQUFBQUtUemd6ZXBEU1lBQUFCZ0FsVk5qTWdEWEFBcjRTeFc2ZWNnLURyX0ZfNHFDV3g3Ni1BU0hR|dd5543bb1f3ae7427298c5649a7f8ebf1e1d5573d87c9ac660fda0a5ba394769"; capsion_ticket="2|1:0|10:1528199822|14:capsion_ticket|44:ZTg4NGU1YWRiNmE4NDRjODk2OWUzYTRmOGViYmM3ZjU=|0a50d3edbb6752cba4d24c1a9e7f253d33d8a5d539bc94ce90c399ab51ace12f"',

'Host':'www.zhihu.com',

'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'

}

r = requests.get('https://www.zhihu.com',headers=headers)
print(r.text)

会话维持
requests.get('http://httpbin.org/cookies/set/number/123456789')
r = requests.get('http://httpbin.org/cookies')

print(r.text)    #{"cookies":{}}

我们在请求测试网址时,设置了一个cookie,名:number,内容:123456789

s = requests.Session()
s.get('http://httpbin.org/cookies/set/number/123456789')
r = s.get('http://httpbin.org/cookies')
print(r.text)
#{"cookies":{"number":"123456789"}}

SSL证书验证
r = requests.get("https://www.12306.cn",verify = False)
这样就避免了证书验证错误,但是报了一个警告
from requesrs.packages import urllib3
urllib3.disable——warnings()
这样就忽略了

还可以指定本地证书用作客户端证书,可以是单个文件(包含密钥和证书),也可以是一个包含两个文件的元组

cert = ("/path/server.crt","/path/key")

本地私有证书一定要解密

代理设置
proxies = {"http":"http://10.10.1.10:3128"}
requests.get("http://www.taobao.com",proxies = proxies)

超时等待;timeout = 1
如果想要永久等待,设置为None,或者不设置,默认

身份认证(简单的只需要账号密码登陆的)
auth = ('username','password')
正确 自动认证成功 200
错误 　　　　　　 401