import requests import json # 1. 基本的请求 ''' r = requests.get('https://github.com/Ranxf') # 不带参数 r1 = requests.get(url='http://dict.baidu.com/s', params={'wd': 'python'}) # 带参数 各种方法: requests.get(‘https://github.com/timeline.json’) # GET请求 requests.post(“http://httpbin.org/post”) # POST请求 requests.put(“http://httpbin.org/put”) # PUT请求 requests.delete(“http://httpbin.org/delete”) # DELETE请求 requests.head(“http://httpbin.org/get”) # HEAD请求 requests.options(“http://httpbin.org/get” ) # OPTIONS请求 ''' # 2. 为 url 传递参数 url_params = {'key': 'value'} r = requests.get('https://www.baidu.com/', params=url_params) print(r.url) # https://www.baidu.com/?key=value # 3. 响应的内容 ''' r.encoding # 获取当前的编码 r.encoding = 'utf-8' # 设置编码 r.text # 以encoding解析返回内容。字符串方式的响应体,会自动根据响应头部的字符编码进行解码。 r.content # 以字节形式(二进制)返回。字节方式的响应体,会自动为你解码 gzip 和 deflate 压缩。 r.headers # 以字典对象存储服务器响应头,但是这个字典比较特殊,字典键不区分大小写,若键不存在则返回None r.status_code # 响应状态码 r.raw # 返回原始响应体,也就是 urllib 的 response 对象,使用 r.raw.read() r.ok # 查看r.ok的布尔值便可以知道是否登陆成功 r.json() # Requests中内置的JSON解码器,以json形式返回,前提返回的内容确保是json格式的,不然解析出错会抛异常 r.raise_for_status() # 失败请求(非200响应)抛出异常 ''' r = requests.post('https://api.github.com/some/endpoint', data=json.dumps({'some': 'data'})) print(r.encoding) # utf-8 print(r.text) # {"message":"Not Found","documentation_url":"https://docs.github.com/rest"} print(r.content) # b'{"message":"Not Found","documentation_url":"https://docs.github.com/rest"}' print(r.json()) # {'message': 'Not Found', 'documentation_url': 'https://docs.github.com/rest'} print(r.ok) # False print(r.status_code) # 404 # 4. 定制头和cookie信息 ''' header = {'user-agent': 'my-app/0.0.1''} cookie = {'key':'value'} r = requests.get/post('your url',headers=header,cookies=cookie) ''' data = {'some': 'data'} headers = {'content-type': 'application/json', 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'} r = requests.post('https://api.github.com/some/endpoint', data=data, headers=headers) print(r.text) # {"message":"Not Found","documentation_url":"https://docs.github.com/rest"} # 5. 响应状态码 ''' 使用requests方法后,会返回一个response对象,其存储了服务器响应的内容,如上实例中已经提到的 r.text、r.status_code…… 获取文本方式的响应体实例:当你访问 r.text 之时,会使用其响应的文本编码进行解码,并且你可以修改其编码让 r.text 使用自定义的编码进行解码。 r = requests.get('http://www.itwhy.org') print(r.text, ' {} '.format('*'*79), r.encoding) r.encoding = 'GBK' print(r.text, ' {} '.format('*'*79), r.encoding) ''' r1 = requests.get(url='http://dict.baidu.com/s', params={'wd': 'python'}) # 带参数的get请求 print(r1.url) # https://dict.baidu.com/s?wd=python print(r1.text) # 网页架构 print(r1.cookies) # <RequestsCookieJar[<Cookie BAIDUID=742D824EDFE46BFDCB825A43F60DD574:FG=1 for .baidu.com/>]> # 6. 响应 ''' r.headers #返回字典类型,头信息 r.requests.headers #返回发送到服务器的头信息 r.cookies #返回cookie r.history #返回重定向信息,当然可以在请求是加上allow_redirects = false 阻止重定向 {'Content-Encoding': 'gzip', 'Content-Type': 'text/html; charset=UTF-8'。。。} {'User-Agent': 'python-requests/2.24.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'} <RequestsCookieJar[<Cookie BAIDUID=24B4BF912AEBE0F59B090535D1E7A4ED:FG=1 for .baidu.com/>]> [<Response [301]>] ''' # 7. 超时 r = requests.get(url='http://dict.baidu.com/s', params={'wd': 'python'}, timeout=1) # 设置秒数超时,仅对于连接有效 # 8. 会话对象,能够跨请求保持某些参数 ''' s = requests.Session() s.auth = ('auth','passwd') s.headers = {'key':'value'} r = s.get('url') r1 = s.get('url1') ''' # 9. 代理 ''' proxies = {'http':'ip1','https':'ip2' } requests.get('url',proxies=proxies) 如果代理需要用户名和密码,则需要这样: proxies = { "http": "http://user:pass@10.10.1.10:3128/", } ''' proxies = { "http": "http://10.10.1.10:3128", "https": "http://10.10.1.100:4444", } r = requests.get('http://m.ctrip.com', proxies=proxies) # 10. request 请求 ''' 10.1 不带参数: ret = requests.get('https://github.com/timeline.json') print(ret.url) # https://github.com/timeline.json print(ret.text) # {"message":"Hello there, wayfaring stranger. If you’re... 10.2 带参数 payload = {'key1': 'value1', 'key2': 'value2'} ret = requests.get("http://httpbin.org/get", params=payload) print(ret.url) # http://httpbin.org/get?key1=value1&key2=value2 print(ret.text) { "args": { "key1": "value1", "key2": "value2" }, "headers": { "Accept": "*/*", "Accept-Encoding": "gzip, deflate", "Host": "httpbin.org", "User-Agent": "python-requests/2.24.0", "X-Amzn-Trace-Id": "Root=1-606c0365-27d2b73c3a50bf4638d8591e" }, "origin": "113.140.16.114", "url": "http://httpbin.org/get?key1=value1&key2=value2" } ''' # 11. POST请求 ''' # 11.1 基本POST实例 import requests payload = {'key1': 'value1', 'key2': 'value2'} ret = requests.post("http://httpbin.org/post", data=payload) print(ret.text) { "args": {}, "data": "", "files": {}, "form": { "key1": "value1", "key2": "value2" }, "headers": { "Accept": "*/*", "Accept-Encoding": "gzip, deflate", "Content-Length": "23", "Content-Type": "application/x-www-form-urlencoded", "Host": "httpbin.org", "User-Agent": "python-requests/2.24.0", "X-Amzn-Trace-Id": "Root=1-606c04af-7a7961387416da906c15c57c" }, "json": null, "origin": "113.140.16.114", "url": "http://httpbin.org/post" } # 11.2 发送请求头和数据实例 import requests import json url = 'https://api.github.com/some/endpoint' payload = {'some': 'data'} headers = {'content-type': 'application/json'} ret = requests.post(url, data=json.dumps(payload), headers=headers) print(ret.text) print(ret.cookies) {"message":"Not Found","documentation_url":"https://docs.github.com/rest"} <RequestsCookieJar[]> 11.3 参数示例 不带参数 def param_method_url(): # requests.request(method='get', url='http://127.0.0.1:8000/test/') # requests.request(method='post', url='http://127.0.0.1:8000/test/') pass 带参数 def param_param(): # - 可以是字典 # - 可以是字符串 # - 可以是字节(ascii编码以内) # requests.request(method='get', # url='http://127.0.0.1:8000/test/', # params={'k1': 'v1', 'k2': '水电费'}) # requests.request(method='get', # url='http://127.0.0.1:8000/test/', # params="k1=v1&k2=水电费&k3=v3&k3=vv3") # requests.request(method='get', # url='http://127.0.0.1:8000/test/', # params=bytes("k1=v1&k2=k2&k3=v3&k3=vv3", encoding='utf8')) # 错误 # requests.request(method='get', # url='http://127.0.0.1:8000/test/', # params=bytes("k1=v1&k2=水电费&k3=v3&k3=vv3", encoding='utf8')) pass 参数类型 def param_data(): # 可以是字典 # 可以是字符串 # 可以是字节 # 可以是文件对象 # requests.request(method='POST', # url='http://127.0.0.1:8000/test/', # data={'k1': 'v1', 'k2': '水电费'}) # requests.request(method='POST', # url='http://127.0.0.1:8000/test/', # data="k1=v1; k2=v2; k3=v3; k3=v4" # ) # requests.request(method='POST', # url='http://127.0.0.1:8000/test/', # data="k1=v1;k2=v2;k3=v3;k3=v4", # headers={'Content-Type': 'application/x-www-form-urlencoded'} # ) # requests.request(method='POST', # url='http://127.0.0.1:8000/test/', # data=open('data_file.py', mode='r', encoding='utf-8'), # 文件内容是:k1=v1;k2=v2;k3=v3;k3=v4 # headers={'Content-Type': 'application/x-www-form-urlencoded'} # ) pass def param_json(): # 将json中对应的数据进行序列化成一个字符串,json.dumps(...) # 然后发送到服务器端的body中,并且Content-Type是 {'Content-Type': 'application/json'} requests.request(method='POST', url='http://127.0.0.1:8000/test/', json={'k1': 'v1', 'k2': '水电费'}) def param_headers(): # 发送请求头到服务器端 requests.request(method='POST', url='http://127.0.0.1:8000/test/', json={'k1': 'v1', 'k2': '水电费'}, headers={'Content-Type': 'application/x-www-form-urlencoded'} ) def param_cookies(): # 发送Cookie到服务器端 requests.request(method='POST', url='http://127.0.0.1:8000/test/', data={'k1': 'v1', 'k2': 'v2'}, cookies={'cook1': 'value1'}, ) # 也可以使用CookieJar(字典形式就是在此基础上封装) from http.cookiejar import CookieJar from http.cookiejar import Cookie obj = CookieJar() obj.set_cookie(Cookie(version=0, name='c1', value='v1', port=None, domain='', path='/', secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False, port_specified=False, domain_specified=False, domain_initial_dot=False, path_specified=False) ) requests.request(method='POST', url='http://127.0.0.1:8000/test/', data={'k1': 'v1', 'k2': 'v2'}, cookies=obj) def param_files(): # 发送文件 # file_dict = { # 'f1': open('readme', 'rb') # } # requests.request(method='POST', # url='http://127.0.0.1:8000/test/', # files=file_dict) # 发送文件,定制文件名 # file_dict = { # 'f1': ('test.txt', open('readme', 'rb')) # } # requests.request(method='POST', # url='http://127.0.0.1:8000/test/', # files=file_dict) # 发送文件,定制文件名 # file_dict = { # 'f1': ('test.txt', "hahsfaksfa9kasdjflaksdjf") # } # requests.request(method='POST', # url='http://127.0.0.1:8000/test/', # files=file_dict) # 发送文件,定制文件名 # file_dict = { # 'f1': ('test.txt', "hahsfaksfa9kasdjflaksdjf", 'application/text', {'k1': '0'}) # } # requests.request(method='POST', # url='http://127.0.0.1:8000/test/', # files=file_dict) pass def param_auth(): # 密码认证 from requests.auth import HTTPBasicAuth, HTTPDigestAuth ret = requests.get('https://api.github.com/user', auth=HTTPBasicAuth('wupeiqi', 'sdfasdfasdf')) print(ret.text) # ret = requests.get('http://192.168.1.1', # auth=HTTPBasicAuth('admin', 'admin')) # ret.encoding = 'gbk' # print(ret.text) # ret = requests.get('http://httpbin.org/digest-auth/auth/user/pass', auth=HTTPDigestAuth('user', 'pass')) # print(ret) def param_timeout(): # ret = requests.get('http://google.com/', timeout=1) # print(ret) # ret = requests.get('http://google.com/', timeout=(5, 1)) # print(ret) pass def param_allow_redirects(): ret = requests.get('http://127.0.0.1:8000/test/', allow_redirects=False) print(ret.text) def param_proxies(): # proxies = { # "http": "61.172.249.96:80", # "https": "http://61.185.219.126:3128", # } # proxies = {'http://10.20.1.128': 'http://10.10.1.10:5323'} # ret = requests.get("http://www.proxy360.cn/Proxy", proxies=proxies) # print(ret.headers) # from requests.auth import HTTPProxyAuth # # proxyDict = { # 'http': '77.75.105.165', # 'https': '77.75.105.165' # } # auth = HTTPProxyAuth('username', 'mypassword') # # r = requests.get("http://www.google.com", proxies=proxyDict, auth=auth) # print(r.text) pass def param_stream(): ret = requests.get('http://127.0.0.1:8000/test/', stream=True) print(ret.content) ret.close() # from contextlib import closing # with closing(requests.get('http://httpbin.org/get', stream=True)) as r: # # 在此处理响应。 # for i in r.iter_content(): # print(i) def requests_session(): import requests session = requests.Session() ### 1、首先登陆任何页面,获取cookie i1 = session.get(url="http://dig.chouti.com/help/service") ### 2、用户登陆,携带上一次的cookie,后台对cookie中的 gpsd 进行授权 i2 = session.post( url="http://dig.chouti.com/login", data={ 'phone': "8615131255089", 'password': "xxxxxx", 'oneMonth': "" } ) i3 = session.post( url="http://dig.chouti.com/link/vote?linksId=8589623", ) print(i3.text) ''' # 12 json请求 ''' #! /usr/bin/python3 import requests import json if __name__ == '__main__': heard = {'Content-Type': 'application/json'} payload = {'CountryName': '中国', 'ProvinceName': '四川省', 'L1CityName': 'chengdu', 'L2CityName': 'yibing', 'TownName': '', 'Longitude': '107.33393', 'Latitude': '33.157131', 'Language': 'CN'} r = requests.post("http://www.xxxxxx.com/CityLocation/json/LBSLocateCity", heards=heard, data=payload) data = r.json() if r.status_code != 200: print('LBSLocateCity API Error' + str(r.status_code)) print(data['CityEntities'][0]['CityID']) # 打印返回json中的某个key的value print(data['ResponseStatus']['Ack']) print( json.dump(data, indent=4, sort_keys=True, ensure_ascii=False)) # 树形打印json,ensure_ascii必须设为False否则中文会显示为unicode ''' # 13. Xml请求 ''' #! /usr/bin/python3 import requests if __name__ == '__main__': heards = {'Content-type': 'text/xml'} XML = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns...>' url = 'http://jobws.push.mobile.xxxxxxxx.com/RefreshWeiXInTokenJob/RefreshService.asmx' r = requests.post(url=url, heards=heards, data=XML) data = r.text print(data) ''' # 14. 状态异常处理 ''' import requests URL = 'http://ip.taobao.com/service/getIpInfo.php' # 淘宝IP地址库API try: r = requests.get(URL, params={'ip': '8.8.8.8'}, timeout=1) r.raise_for_status() # 如果响应状态码不是 200,就主动抛出异常 except requests.RequestException as e: print(e) else: result = r.json() print(type(result), result, sep=' ') ''' # 15. 上传文件 ''' import requests url = 'http://127.0.0.1:8080/upload' files = {'file': open('/home/rxf/test.jpg', 'rb')} # files = {'file': ('report.jpg', open('/home/lyb/sjzl.mpg', 'rb'))} #显式的设置文件名 # 把字符串当成文件上传 # files = {'file': ('test.txt', b'Hello Requests.')} #必需显式的设置文件名 r = requests.post(url, files=files) print(r.text) ''' # 16. 身份验证 ''' import requests from requests.auth import HTTPBasicAuth r = requests.get('https://httpbin.org/hidden-basic-auth/user/passwd', auth=HTTPBasicAuth('user', 'passwd')) # r = requests.get('https://httpbin.org/hidden-basic-auth/user/passwd', auth=('user', 'passwd')) # 简写 print(r.json()) # {'authenticated': True, 'user': 'user'} # 另外一种 # requests.get(URL, auth=HTTPDigestAuth('user', 'pass')) ''' # 17. Cookies与会话对象 ''' import requests r = requests.get('https://www.baidu.com/') print(r.cookies) # <RequestsCookieJar[<Cookie BDORZ=27315 for .baidu.com/>]> print(r.cookies['BDORZ']) # 27315 print(tuple(r.cookies)) # (Cookie(version=0, name='BDORZ', value='27315'... 要想发送你的cookies到服务器,可以使用 cookies 参数: url = 'http://httpbin.org/cookies' cookies = {'testCookies_1': 'Hello_Python3', 'testCookies_2': 'Hello_Requests'} # 在Cookie Version 0中规定空格、方括号、圆括号、等于号、逗号、双引号、斜杠、问号、@,冒号,分号等特殊符号都不能作为Cookie的内容。 r = requests.get(url, cookies=cookies) print(r.json()) ''' # 18. requests模块抓取网页源码并保存到文件示例 ''' import requests # requests模块抓取网页源码并保存到文件示例 html = requests.get("http://www.baidu.com") with open('test.txt', 'w', encoding='utf-8') as f: f.write(html.text) # 读取一个txt文件,每次读取一行,并保存到另一个txt文件中的示例 ff = open('testt.txt', 'w', encoding='utf-8') with open('test.txt', encoding="utf-8") as f: for line in f: ff.write(line) ff.close() ''' # 19. 自动登录示例1: ''' import requests from bs4 import BeautifulSoup def login_github(): # 通过requests模块模拟浏览器登陆GitHub # 获取csrf_token r1 = requests.get('https://github.com/login') # 获得get请求的对象 s1 = BeautifulSoup(r1.text, 'html.parser') # 使用bs4解析HTML对象 token = s1.find('input', attrs={'name': 'authenticity_token'}).get('value') # 获取登陆授权码,即csrf_token get_cookies = r1.cookies.get_dict() # 获取get请求的cookies,post请求时必须携带 # 发送post登陆请求 r2 = requests.post( 'https://github.com/session', data={ 'commit': 'Sign+in', 'utf8': '✓', 'authenticity_token': token, 'login': 'JackUpDown', 'password': '**********' }, cookies=get_cookies # 携带get请求的cookies ) login_cookies = r2.cookies.get_dict() # 获得登陆成功的cookies,携带此cookies就可以访问任意GitHub页面 # 携带post cookies跳转任意页面 r3 = requests.get('https://github.com/settings/emails', cookies=login_cookies) print(r3.text) if __name__ == '__main__': login_github() ''' # 20. 自动登录示例2: # ############## 方式一 ############## """ # ## 1、首先登陆任何页面,获取cookie i1 = requests.get(url="http://dig.chouti.com/help/service") i1_cookies = i1.cookies.get_dict() # ## 2、用户登陆,携带上一次的cookie,后台对cookie中的 gpsd 进行授权 i2 = requests.post( url="http://dig.chouti.com/login", data={ 'phone': "8615131255089", 'password': "xxooxxoo", 'oneMonth': "" }, cookies=i1_cookies ) # ## 3、点赞(只需要携带已经被授权的gpsd即可) gpsd = i1_cookies['gpsd'] i3 = requests.post( url="http://dig.chouti.com/link/vote?linksId=8589523", cookies={'gpsd': gpsd} ) print(i3.text) """ # ############## 方式二 ############## """ import requests session = requests.Session() i1 = session.get(url="http://dig.chouti.com/help/service") i2 = session.post( url="http://dig.chouti.com/login", data={ 'phone': "8615131255089", 'password': "xxooxxoo", 'oneMonth': "" } ) i3 = session.post( url="http://dig.chouti.com/link/vote?linksId=8589523" ) print(i3.text) """ # 21. 自动登录示例3:知乎 import time import requests from bs4 import BeautifulSoup session = requests.Session() i1 = session.get( url='https://www.zhihu.com/#signin', headers={ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36', } ) soup1 = BeautifulSoup(i1.text, 'lxml') xsrf_tag = soup1.find(name='input', attrs={'name': '_xsrf'}) xsrf = xsrf_tag.get('value') current_time = time.time() i2 = session.get( url='https://www.zhihu.com/captcha.gif', params={'r': current_time, 'type': 'login'}, headers={ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36', }) with open('zhihu.gif', 'wb') as f: f.write(i2.content) captcha = input('请打开zhihu.gif文件,查看并输入验证码:') form_data = { "_xsrf": xsrf, 'password': 'xxooxxoo', "captcha": 'captcha', 'email': '424662508@qq.com' } i3 = session.post( url='https://www.zhihu.com/login/email', data=form_data, headers={ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36', } ) i4 = session.get( url='https://www.zhihu.com/settings/profile', headers={ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36', } ) soup4 = BeautifulSoup(i4.text, 'lxml') tag = soup4.find(id='rename-section') nick_name = tag.find('span', class_='name').string print(nick_name) # 22 自动登录4 博客园 import re import json import base64 import rsa import requests def js_encrypt(text): b64der = 'MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCp0wHYbg/NOPO3nzMD3dndwS0MccuMeXCHgVlGOoYyFwLdS24Im2e7YyhB0wrUsyYf0/nhzCzBK8ZC9eCWqd0aHbdgOQT6CuFQBMjbyGYvlVYU2ZP7kG9Ft6YV6oc9ambuO7nPZh+bvXH0zDKfi02prknrScAKC0XhadTHT3Al0QIDAQAB' der = base64.standard_b64decode(b64der) pk = rsa.PublicKey.load_pkcs1_openssl_der(der) v1 = rsa.encrypt(bytes(text, 'utf8'), pk) value = base64.encodebytes(v1).replace(b' ', b'') value = value.decode('utf8') return value session = requests.Session() i1 = session.get('https://passport.cnblogs.com/user/signin') rep = re.compile("'VerificationToken': '(.*)'") v = re.search(rep, i1.text) verification_token = v.group(1) form_data = { 'input1': js_encrypt('wptawy'), 'input2': js_encrypt('asdfasdf'), 'remember': False } i2 = session.post(url='https://passport.cnblogs.com/user/signin', data=json.dumps(form_data), headers={ 'Content-Type': 'application/json; charset=UTF-8', 'X-Requested-With': 'XMLHttpRequest', 'VerificationToken': verification_token} ) i3 = session.get(url='https://i.cnblogs.com/EditDiary.aspx') print(i3.text)