爬虫 --- 04. 代理服务器, 验证码识别, 处理cookie,线程池

一.代理服务器

　　1.基本概念

- 代理的网站：
    - 快代理
    - 西祠代理
    - goubanjia  (概率较高)
　　　　- 代理知识：https://help.kuaidaili.com/wiki/
- 匿名度：
    - 透明：对方服务器可以知道你使用了代理，并且也知道你的真实IP
    - 匿名：对方服务器可以知道你使用了代理，但不知道你的真实IP
    - 高匿：对方服务器不知道你使用了代理，更不知道你的真实IP。
- 类型：
    - http：该类型的代理ip只可以发起http协议头对应的请求
    - https：该类型的代理ip只可以发起https协议头对应的请求

　　2.示例

#简单版

import requests

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
url = 'https://www.baidu.com/s?wd=ip'


page_text = requests.get(url=url,headers=headers,proxies={"https":"101.80.87.77:9000"}).text

with open('./ip.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

#升级版(减少被封的几率)

import requests
import random

https = [
    {'https':'223.19.212.30:8380'},
    {'https':'221.19.212.30:8380'}
]
http = [
    {'http':'223.19.212.30:8380'},
    {'http':'221.19.212.30:8380'}
]

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
url = 'https://www.baidu.com/s?wd=ip'

if url.split(':')[0] == 'https':
    page_text = requests.get(url=url,headers=headers,proxies=random.choice(https)).text
else:
     page_text = requests.get(url=url,headers=headers,proxies=random.choice(http)).text

with open('./ip.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

 二.验证码识别

常用的接口网站

　　1.云打码，
　　2.打码兔，
　　3.超级鹰

云打码使用流程
　　http://www.yundama.com/demo.html
　　注册：
　　　　普通用户
　　　　开发者用户
　　登录：
　　　　登录普通用户：
　　　　　　查询剩余题分
　　　　登录开发者用户：
　　　　　　创建一个软件：我的软件-》创建一个新软件（软件名称，秘钥不可以修改），使用软件的id和秘钥
　　　　　　下载示例代码：开发文档-》点此下载：云打码接口DLL-》PythonHTTP示例下载

以下代码是下载的文件中的内容

　　用的时候需加载

import http.client, mimetypes, urllib, json, time, requests

######################################################################

class YDMHttp:

    apiurl = 'http://api.yundama.com/api.php'
    username = ''
    password = ''
    appid = ''
    appkey = ''

    def __init__(self, username, password, appid, appkey):
        self.username = username  
        self.password = password
        self.appid = str(appid)
        self.appkey = appkey

    def request(self, fields, files=[]):
        response = self.post_url(self.apiurl, fields, files)
        response = json.loads(response)
        return response
    
    def balance(self):
        data = {'method': 'balance', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['balance']
        else:
            return -9001
    
    def login(self):
        data = {'method': 'login', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['uid']
        else:
            return -9001

    def upload(self, filename, codetype, timeout):
        data = {'method': 'upload', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'codetype': str(codetype), 'timeout': str(timeout)}
        file = {'file': filename}
        response = self.request(data, file)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['cid']
        else:
            return -9001

    def result(self, cid):
        data = {'method': 'result', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid)}
        response = self.request(data)
        return response and response['text'] or ''

    def decode(self, filename, codetype, timeout):
        cid = self.upload(filename, codetype, timeout)
        if (cid > 0):
            for i in range(0, timeout):
                result = self.result(cid)
                if (result != ''):
                    return cid, result
                else:
                    time.sleep(1)
            return -3003, ''
        else:
            return cid, ''

    def report(self, cid):
        data = {'method': 'report', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid), 'flag': '0'}
        response = self.request(data)
        if (response):
            return response['ret']
        else:
            return -9001

    def post_url(self, url, fields, files=[]):
        for key in files:
            files[key] = open(files[key], 'rb');
        res = requests.post(url, files=files, data=fields)
        return res.text

#将示例代码中的可执行程序封装成函数
def transformCodeImg(imgPath,imgType):
    # 普通用户名
    username    = 'bobo328410948'

    # 密码
    password    = 'bobo328410948'                            

    # 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！
    appid       = 6003                                     

    # 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！
    appkey      = '1f4b564483ae5c907a1d34f8e2f2776c'    

    # 图片文件
    filename    = imgPath                        

    # 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html
    codetype    = imgType

    # 超时时间，秒
    timeout     = 30                                    
    result = None
    # 检查
    if (username == 'username'):
        print('请设置好相关参数再测试')
    else:
        # 初始化
        yundama = YDMHttp(username, password, appid, appkey)

        # 登陆云打码
        uid = yundama.login();
        print('uid: %s' % uid)

        # 查询余额
        balance = yundama.balance();
        print('balance: %s' % balance)

        # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果
        cid, result = yundama.decode(filename, codetype, timeout);
        
    return result
    

 三.处理cookie

- cookie：可是使得服务器端记录客户端的相关状态

- 处理cookie的方法:
　　　　-手动处理:  在hears中加入cookie,但cookie是有有效时长的
　　　　-自动处理:　使用会话对象Session
- Session的用法:
　　　　- 实例化一个会话对象: requests.Session()
　　　　- session的作用:
　　　　　　　　- 会话对象可以进行请求发送(post,get)
　　　　　　　　-  如果在进行请求发送过程中产生的cookie,会被自动存在session中

#需求：爬取雪球网中的新闻标题和对应的内容简介

import requests
headers = {

    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}

#实例化对象
sesion = requests.Session()
#这一步如果产生cookie,则cookie会被自动存储到session中
session.get(url="https://xueqiu.com/",headers=headers)

url = 'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=-1&count=10&category=-1'#session对象中就携带了cookie,发送请求
json_obj = session.get(url=url,headers=headers).json()
print(json_obj)

四.综合示例(验证码,模拟登陆,cookie)

#对古诗文网进行模拟登陆
#1.获取验证码图片,并且让打码平台进行识别

import requests
from lxml import etree


s = requests.Session()

url = "https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx"

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
page_text = s.get(url= url,headers= headers).text
tree = etree.HTML(page_text)
img_src = "https://so.gushiwen.org" + tree.xpath('//*[@id="imgCode"]/@src')[0]

img_data = s.get(url=img_src,headers=headers).content

with open("./gushiwen.jpg","wb") as fp:
    fp.write(img_data)
    
result= transformCodeImg("./gushiwen.jpg",1004)
print(result)

__VIEWSTATE = tree.xpath('//*[@id="__VIEWSTATE"]/@value')[0]
__VIEWSTATEGENERATOR = tree.xpath('//*[@id="__VIEWSTATEGENERATOR"]/@value')[0]

#模拟登陆

post_url = "https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx"

data = {
    "__VIEWSTATE": __VIEWSTATE,
    "__VIEWSTATEGENERATOR": __VIEWSTATEGENERATOR,
    "from": "http://so.gushiwen.org/user/collect.aspx",
    "email": "www.zhangbowudi@qq.com",
    "pwd": "bobo328410948",
    "code": result,
    "denglu": "登录",
}
response = s.post(url= post_url,data= data, headers=headers)

print(response.status_code)

page_text = response.text
with open("./gushi.html","w",encoding="utf-8") as fp:
    fp.write(page_text)

 五.线程池的使用

线程池需要用到爬虫较为耗时的操作中

    耗时的操作:
        视频的下载,
        视频的保存

import re
from multiprocessing.dummy import Pool
pool = Pool(4)　　　　#创建线程池

url = "https://www.pearvideo.com/category_1"

page_text = requests.get(url=url, headers= headers).text

tree = etree.HTML(page_text)

li_list = tree.xpath('//ul[@id="listvideoListUl"]/li')


video_urls = []  #存放所有的视频url
for li in li_list:
    deta_url ='https://www.pearvideo.com/'+ li.xpath('./div/a/@href')[0]
    
    detail_page = requests.get(url= deta_url,headers= headers).text
    
    ex = 'srcUrl="(.*?)",vdoUrl='
    
    videl_src = re.findall(ex,detail_page,re.S)[0]
    video_urls.append(videl_src)
#使用异步(线程池)下载视频
all_video = pool.map(getVidelData,video_urls)  #前参数为函数,后参数为列表
print("下载完成")
使用异步保存视频
pool.map(saveVideo,all_video)

print("保存完成")

#下载视频的函数
def getVidelData(url):
    return requests.get(url=url,headers=headers).content

#保存视频的函数
import random

def saveVideo(data):
    name = str(random.randint(0,9999))+".mp4"
    with open(name,"wb") as fp:
        fp.write(data)