Python 爬取 ajax 返回的数据

from urllib.request import quote, unquote
import random
import requests

#quote 将单个字符串编码转化为 %xx 的形式
# strip() 方法用于移除字符串头尾指定的字符（默认为空格或换行符）或字符序列
baidu_cat = quote('总榜').strip();

refer_url = 'https://data.wxb.com/rankArticle'
ajax_url = 'https://data.wxb.com/rank/article?baidu_cat=%s&baidu_tag=&page=1&pageSize=50&type=2&order='%baidu_cat

headers = {
    'Accept': 'application/json',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Connection': 'keep-alive',
    'Host': 'data.wxb.com',
    'Referer': refer_url,
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
}

try:
    resp = requests.get(ajax_url, headers=headers)
    if resp.status_code == 200:
        print(resp.json())  #解析内容为json返回
except requests.ConnectionError as e:
    print('Error',e.args) #输出异常信息

result = resp.json()
for item in result['data']:
    print('url:',item['url'])
    print('title:',item['title'])

print("程序结束")



''' 
pip install requests[security]

这样会额外的安装如下3个包:
pyOpenSSL
cryptography
idna

'''