requests

GET请求

import requests

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
}
data = {
    'wd': '码云'
}
url = "https://www.baidu.com/s"
response = requests.get(url, headers=headers, params=data, verify=False)
print(response.status_code)
print(response.request.url)
print(response.content.decode())

POST请求

import json

import requests
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
headers = {
    'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Mobile Safari/537.36'

}
query_string = input("输入你要翻译的汉字：")
data = {
    'query': query_string,
    'from': 'zh',
    'to': 'en'

}
url = 'https://fanyi.baidu.com/basetrans'
response = requests.post(url=url, headers=headers, data=data, verify=False)
# print(response.content.decode())

dict_ret = json.loads(response.content.decode())
ret = dict_ret['trans'][0]['dst']
print(ret)

session登录

import requests
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
session = requests.Session()

url = "https://login.m.taobao.com/login.htm"
headers = {
    'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'
}
data = {
    'TPL_username': "xxx",
    'TPL_password': "xxx"
}
session.post(url=url, headers=headers, data=data, verify=False)
ret = session.get("https://h5.m.taobao.com/mlapp/olist.html", headers=headers, verify=False)
with open('cart.html', 'w', encoding='utf-8') as f:
    f.write(ret.content.decode('utf-8'))

cookie登录

import requests
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

headers = {
    'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
    'Cookie': 'isg=BHx8iHc6DAlbjzjWCQinbw9ITRru3SevFeYCa1b9iGdKIRyrfoXwL_KTBc9Zclj3'
}

ret = requests.get("https://h5.m.taobao.com/mlapp/olist.html", headers=headers, verify=False)
with open('cart.html', 'w', encoding='utf-8') as f:
    f.write(ret.content.decode('utf-8'))

cookie生成键值对

import requests
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
url = 'https://h5.m.taobao.com/mlapp/olist.html'
headers = {
    'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
}

cookies = 'thw=cn; t=d04efea3a8fd36e76208d492e2af8f43; '
cookies = {i.split('=')[0]: i.split('=')[1] for i in cookies.split("; ")}

print(cookies)
ret = requests.get(url, headers=headers, cookies=cookies, verify=False)
with open('cart.html', 'w', encoding='utf-8') as f:
    f.write(ret.content.decode('utf-8'))

cookiejar

import requests
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

url = "https://www.baidu.com/"
headers = {
    'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'
}
response = requests.get(url, headers=headers, verify=False)

# 将CookieJar转为字典：
cookies_dict = requests.utils.dict_from_cookiejar(response.cookies)
print(cookies_dict)
# 将字典转为CookieJar：
# cookie_dict - 插入到CookieJar中的键/值的字典。
# cookiejar - （可选）一个cookiejar来添加cookie。
# 覆盖 - （可选）如果为False，则不会使用新的jar替换jar中已有的cookie。
cookies = requests.utils.cookiejar_from_dict(cookies_dict, cookiejar=None, overwrite=True)
print(cookies)

# 转换完之后就可以把它赋给cookies 并传入到session中了：
# s = requests.Session()
# s.cookies = cookies
#
# print(s.cookies)

url地址解码

import requests
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

url = 'https://www.baidu.com/s?word=人工智能'

ret = requests.utils.quote(url)
print(ret)

ret = requests.utils.unquote(ret)
print(ret)

headers = {
    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'
}
response = requests.get(url, timeout=5, headers=headers, verify=False)
print(response.content.decode())

异常重试

import requests
from retrying import retry

url = "http://www.youtube.com"
headers = {
    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'
}


# 最大重试3次 异常捕获
@retry(stop_max_attempt_number=3)
def _parse_url(urls, method, data, proxies):
    if method == 'POST':
        response = requests.post(urls, headers=headers, data=data, timeout=3, proxies=proxies)
    else:
        response = requests.get(urls, headers=headers, data=data, timeout=3, proxies=proxies)
    assert response.status_code == 200
    return response.content.decode()


def parse_url(urls, method='get', data=None, proxies={}):
    try:
        html_str = _parse_url(urls, method, data, proxies)
    except:
        html_str = None

    return html_str


if __name__ == '__main__':
    print(parse_url(url))