使用python的requests模块采集请求中的数据

# coding=gbk

# Create your tests here.

import requests, re, json, math

# 设置全局session变量
r_session = requests.session()

#登录账户
username = 'admin'

#登录密码
password = '111111'

#登录ip
ip = 'localhost'

#登录port
port = '8088'

#模型总数
num = 1679

#调用登录接口，向session中插入登录状态
def login():
    login_headers = {
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Content-Length': '40',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Host': '{ip}:{port}'.format(ip=ip, port=port),
        'Origin': 'http://{ip}:{port}'.format(ip=ip, port=port),
        'Referer': 'http://{ip}:{port}/login'.format(ip=ip, port=port),
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest'
    }#设置登录请求头
    login_data = {"username": username, "password": password}#设置登录请求体
    r = r_session.post('http://{ip}:{port}/login'.format(ip=ip, port=port), headers=login_headers, data=login_data)#发送登录请求

#获取系统中的所有模型id
def get_model_id(num):
    modelnum = math.ceil(num/100)
    model_id = []#用于存储所有的模型id
    model_headers = {'Content-Type': 'application/json'}
    for x in range(modelnum):
        x *= 100
        r = r_session.get(
            "http://{}:{}/model/list?limit=100&offset={}&sortName=updatetime&sortOrder=desc&name=&id=&status=".format(ip, port,
                x), headers=model_headers).text
        r = json.loads(r)
        rows = r.get("rows")
        for i in rows:
            model_id.append(i.get("id"))
    return model_id

#获取所有的错误信息
def get_error(model_id):
    repeat_id = []#用于存储所有的错误信息
    for model_id_detail in model_id:
        model_detail = r_session.get('http://{ip}:{port}/model/modify/'.format(ip=ip, port=port) + str(model_id_detail)).text
        edbmodel = re.findall(r'^<input id="edbModel" type="hidden" value="(.*?)" />$', model_detail, re.M)
        edbmodel = edbmodel[0]
        edbmodel = edbmodel.replace('&quot;', '').replace(',,', ',').replace('\', '')
        edbmodel = re.findall(r'indicatorId:(?P<indicatorId>d*?),indicatorName:.*?[(?P<indicatorName>d*?)]', edbmodel, re.M)
        for model in edbmodel:
            if model[0] != model[1]:
                model_error = str(model_id_detail) #+ ':' + str(model[0]) + ':' + str(model[1])#将错误模型、错误指标id、正确指标id进行拼接
                repeat_id.append(model_error)
    print(repeat_id)#打印错误信息
    print(len(repeat_id))#打印错误信息长度

if __name__ == '__main__':
    login()#将登录状态保存在session中
    model_id = get_model_id(num)#将所有的模型id以列表的形式保存在model_id中
    get_error(model_id)#获取错误信息