基于cookie登录+验证码如何爬取

实例：

需求：获取人人网用户登录过后的个人主页数据

#云打码平台登录，直接下载引用就好，无需更改
import http.client, mimetypes, urllib, json, time, requests
class YDMHttp:

    apiurl = 'http://api.yundama.com/api.php'
    username = ''
    password = ''
    appid = ''
    appkey = ''

    def __init__(self, username, password, appid, appkey):
        self.username = username  
        self.password = password
        self.appid = str(appid)
        self.appkey = appkey

    def request(self, fields, files=[]):
        response = self.post_url(self.apiurl, fields, files)
        response = json.loads(response)
        return response
    
    def balance(self):
        data = {'method': 'balance', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['balance']
        else:
            return -9001
    
    def login(self):
        data = {'method': 'login', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['uid']
        else:
            return -9001

    def upload(self, filename, codetype, timeout):
        data = {'method': 'upload', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'codetype': str(codetype), 'timeout': str(timeout)}
        file = {'file': filename}
        response = self.request(data, file)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['cid']
        else:
            return -9001

    def result(self, cid):
        data = {'method': 'result', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid)}
        response = self.request(data)
        return response and response['text'] or ''

    def decode(self, filename, codetype, timeout):
        cid = self.upload(filename, codetype, timeout)
        if (cid > 0):
            for i in range(0, timeout):
                result = self.result(cid)
                if (result != ''):
                    return cid, result
                else:
                    time.sleep(1)
            return -3003, ''
        else:
            return cid, ''

    def report(self, cid):
        data = {'method': 'report', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid), 'flag': '0'}
        response = self.request(data)
        if (response):
            return response['ret']
        else:
            return -9001

    def post_url(self, url, fields, files=[]):
        for key in files:
            files[key] = open(files[key], 'rb');
        res = requests.post(url, files=files, data=fields)
        return res.text

def parse_codeImg(imgPath):
    # 用户名
    username    = 'xxxx'#自己注册的用户名

    # 密码
    password    = 'xxxx'    #自己注册的密码                     

    # 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！
    appid       = 6372                                     

    # 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！
    appkey      = '9b672eb204d7eede7ddeda5a87d7be08'    

    # 图片文件
    filename    = imgPath                       

    # 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html
    codetype    = 2004

    # 超时时间，秒
    timeout     = 30                                    

    # 检查
    if (username == 'username'):
        print('请设置好相关参数再测试')
    else:
        # 初始化
        yundama = YDMHttp(username, password, appid, appkey)

        # 登陆云打码
        uid = yundama.login();
        print('uid: %s' % uid)

        # 查询余额
        balance = yundama.balance();
        print('balance: %s' % balance)

        # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果
        cid, result = yundama.decode(filename, codetype, timeout);
        print('cid: %s, result: %s' % (cid, result))
        return result

import requests
from lxml import etree
import json
import time
#创建一个session对象，会自动保存cookie
session=requests.session()
#获取人人网URL
url='http://www.renren.com'
#仿造headers
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
}
page_text=requests.get(url=url,headers=headers).text
#解析验证码图片，保存到本地
tree=etree.HTML(page_text)
code_img_url=tree.xpath('//img[@id="verifyPic_login"]/@src')[0]
code_img_data=requests.get(url=code_img_url,headers=headers).content
with open('./code.png','wb') as fp:
    fp.write(code_img_data)
    print("验证码存储成功！！")
code_text=parse_codeImg('./code.png')
print(code_text)
#登录操作，获取cookie
#此url以及data数据需要用Fidder工具抓包获取
login_url="http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=20181131725329"
data={
    "email":"18526303496",
    "icode":code_text,
    "origURL":"http://www.renren.com/home",
    "domain":"renren.com",
    "key_id":"1",
    "captcha_type":"web_login",
    "password":"3f06abf49c06d3f2dfce6554f070677f2459a14159d738eb08f8f7922280f5b7",
    "rkey":"3ca02f6d93a15caf7d0c0b3637abf5a8",
    "f":'http%3A%2F%2Fwww.renren.com%2F969092014'   
}
#使用session发起请求，将cookie存储到session，保证请求成功，
session.post(url=login_url,headers=headers,data=data)

#进行个人主页页面的数据爬取
personoal_url='http://www.renren.com/969092014/profile'
page_text=session.get(url=personoal_url,headers=headers).text
with open('./renren.html','w',encoding='utf-8')as fp:
    fp.write(page_text)
    print('over')