requests抓取数据报错
- HttpConnectinPool: - 原因： - 1.短时间内发起了高频的请求导致ip被禁 - 2.http连接池中的连接资源被耗尽 - 解决： - 1.代理 - 2.headers中加入Conection：“close”
代理服务器
- 代理：代理服务器，可以接受请求然后将其转发。 - 匿名度 - 高匿：既不知道请求者使用了代理，也不知道请求者的真实IP - 匿名：知道请求者使用了代理，但是不知道请求者的真实IP - 透明：知道请求者使用了代理并且知道请求者的真实IP - 类型： - http - https - 免费代理： - www.goubanjia.com - 快代理 - 西祠代理 - http://http.zhiliandaili.cn/
在requests.get()方法中使用代理IP
import requests headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36' } url = 'https://www.baidu.com/s?wd=ip' page_text = requests.get(url,headers=headers,proxies={'https':'111.231.94.44:8888'}).text with open('ip.html','w',encoding='utf-8') as fp: fp.write(page_text)
手动生成代理池
import random proxy_list = [ {'https':'121.231.94.44:8888'}, {'https':'131.231.94.44:8888'}, {'https':'141.231.94.44:8888'} ] url = 'https://www.baidu.com/s?wd=ip' page_text = requests.get(url,headers=headers,proxies=random.choice(proxy_list)).text with open('ip.html','w',encoding='utf-8') as fp: fp.write(page_text)
从网上抓取代理IP自动生成代理池
from lxml import etree import random #从代理精灵中提取代理ip（用于爬取免费代理IP的代理IP是付费的） ip_url = 'http://t.11jsq.com/index.php/api/entry?method=proxyServer.generate_api_url&packid=1&fa=0&fetch_key=&groupid=0&qty=4&time=1&pro=&city=&port=1&format=html&ss=5&css=&dt=1&specialTxt=3&specialJson=&usertype=2' page_text = requests.get(ip_url,headers=headers).text tree = etree.HTML(page_text) ip_list = tree.xpath('//body//text()') print(ip_list) #爬取西祠代理 headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36', 'Connection':"close" } url = 'https://www.xicidaili.com/nn/%d' proxy_list_http = [] proxy_list_https = [] for page in range(1,20): new_url = format(url%page) ip_port = random.choice(ip_list) page_text = requests.get(new_url,headers=headers,proxies={'https':ip_port}).text tree = etree.HTML(page_text) #tbody不可以出现在xpath表达式中 tr_list = tree.xpath('//*[@id="ip_list"]//tr')[1:] for tr in tr_list: ip = tr.xpath('./td[2]/text()')[0] port = tr.xpath('./td[3]/text()')[0] t_type = tr.xpath('./td[6]/text()')[0] ips = ip+':'+port if t_type == 'HTTP': dic = { t_type: ips } proxy_list_http.append(dic) else: dic = { t_type:ips } proxy_list_https.append(dic) print(len(proxy_list_http),len(proxy_list_https)) #检测代理IP是否可用 for ip in proxy_list_http: response = requests.get('https://www/sogou.com',headers=headers,proxies={'https':ip}) if response.status_code == '200': print('检测到了可用ip')
Cookie
- cookie的处理 - 手动处理：将cookie封装到headers中 - 自动处理：session对象。可以创建一个session对象，改对象可以像requests一样进行请求发送。不同之处在于如果在使用session进行请求发送的过程中产生了cookie，则cookie会被自动存储在session对象中。
示例1.1（不携带Cookie访问）
import requests #对雪球网中的新闻数据进行爬取https://xueqiu.com/ url="https://xueqiu.com/" headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36', } response_text=requests.get(url,headers).text response_text
此时是获取不到网页的数据信息，因为如果想要访问页面的数据，需要携带Cookie数据。
示例1.2（手动添加Cookie后访问）
import requests url="https://xueqiu.com/" headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36', 'Cookie':"aliyungf_tc=AQAAAG6X1z3opwMAkLryeKCukrQNV62H; acw_tc=2760822e15886875892523578ed4228020edfe26c3c0eb41d7d9467d8bf6e3; xq_a_token=48575b79f8efa6d34166cc7bdc5abb09fd83ce63; xqat=48575b79f8efa6d34166cc7bdc5abb09fd83ce63; xq_r_token=7dcc6339975b01fbc2c14240ce55a3a20bdb7873; xq_id_token=eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJ1aWQiOi0xLCJpc3MiOiJ1YyIsImV4cCI6MTU4OTY4MjczMCwiY3RtIjoxNTg4Njg3NTYzMDY1LCJjaWQiOiJkOWQwbjRBWnVwIn0.oXNGRbTOZfgChAFNq-BN9v7Q01-ogPgYI-nNDdasJKwSIF4TpfPgTZzRQ6evFHxCmX22GvrL-N7nCVwYTnWWn-7oB7K9d6dagYPja5uWqBNwI1qL7A5yP_SF4OG0meC2BSOU-gAt7whoE7DC-ChkJL0CJ5ZyqjNnYsl_EJjPUDMvEm0ex6surEHJW3uIfh15iIUYJKrjT5FxxjkyNe_C0KjIZXRgJMK77-rcTxlBxzHJkeCIsEKwpEYjKTWAJJYL4r-gC49wJvT_Y2WrdVOtQ9rXT2Q2_rHStT-zEBb9p55ZZakfHb9uzFadI7J1Zkl6w02ns8DVt-DKKRM5XRBg3A; u=691588687589257; Hm_lvt_1db88642e346389874251b5a1eded6e3=1588687591; device_id=24700f9f1986800ab4fcc880530dd0ed; s=co11ch62mg; __utma=1.206451581.1588687610.1588687610.1588687610.1; __utmc=1; __utmz=1.1588687610.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmt=1; __utmb=1.1.10.1588687610; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1588687614", } response_text=requests.get(url,headers=headers).text response_text=response_text.encode("iso-8859-1").decode("utf-8") response_text
此时可以获得页面数据信息，但是如果目标网站每次访问的Cookie是动态生成的，手动添加就行不通了。
示例1.3（使用Session对象自动获取并添加Cookie到请求信息中）
import requests url="https://xueqiu.com/" headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36', } session=requests.Session() response_text=session.get(url,headers=headers).text response_text=response_text.encode("iso-8859-1").decode("utf-8") response_text
此时也是能够顺利取到网页数据。
自动登录中的图片验证码识别
- 验证码的识别 - 超级鹰：http://www.chaojiying.com/about.html - 注册：（用户中心身份） - 登陆： - 创建一个软件：899370 - 下载示例代码 - 打码兔 - 云打码
古诗文网登录图片验证码识别
#!/usr/bin/env python # coding:utf-8 import requests from hashlib import md5 class Chaojiying_Client(object): def __init__(self, username, password, soft_id): self.username = username password = password.encode('utf-8') self.password = md5(password).hexdigest() self.soft_id = soft_id self.base_params = { 'user': self.username, 'pass2': self.password, 'softid': self.soft_id, } self.headers = { 'Connection': 'Keep-Alive', 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)', } def PostPic(self, im, codetype): """ im: 图片字节 codetype: 题目类型 参考 http://www.chaojiying.com/price.html """ params = { 'codetype': codetype, } params.update(self.base_params) files = {'userfile': ('ccc.jpg', im)} r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers) return r.json() def ReportError(self, im_id): """ im_id:报错题目的图片ID """ params = { 'id': im_id, } params.update(self.base_params) r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers) return r.json()
# 返回验证码文本 def transformImg(imgPath,type_code): chaojiying = Chaojiying_Client('15922471244', 'sun10387834...', '904968') im = open(imgPath, 'rb').read() return chaojiying.PostPic(im, type_code)
# 古诗文网验证码识别 from lxml import etree url="https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx" response_text=requests.get(url,headers=headers).text tree=etree.HTML(response_text) img_src="https://so.gushiwen.org"+tree.xpath('//img[@id="imgCode"]/@src')[0] img_data=requests.get(url=img_src,headers=headers).content with open("./code.jpg","wb") as fp: fp.write(img_data) code_text=transformImg("./code.jpg",1902) print(code_text)
模拟登陆古诗文网
# 模拟登陆 from lxml import etree import requests session=requests.Session() url="https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx" response_text=session.get(url,headers=headers).text tree=etree.HTML(response_text) img_src="https://so.gushiwen.org"+tree.xpath('//img[@id="imgCode"]/@src')[0] img_data=session.get(url=img_src,headers=headers).content with open("./code.jpg","wb") as fp: fp.write(img_data) code_text=transformImg("./code.jpg",1902)["pic_str"] __VIEWSTATE=tree.xpath('//input[@id="__VIEWSTATE"]/@value')[0] __VIEWSTATEGENERATOR=tree.xpath('//input[@id="__VIEWSTATEGENERATOR"]/@value')[0] # print(code_text) # print(__VIEWSTATE) # print(__VIEWSTATEGENERATOR) login_url="https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx" data={ "__VIEWSTATE":__VIEWSTATE, "__VIEWSTATEGENERATOR": __VIEWSTATEGENERATOR, 'from':" http://so.gushiwen.org/user/collect.aspx", "email": "15922471244", "pwd": "sun10387834...", "code": code_text, "denglu": "登录", } response_content=session.post(login_url,data=data,headers=headers).text with open("./gushiwen.html","w",encoding="utf-8") as fp: fp.write(response_content)
模拟登陆经验总结：
常规的模拟登陆网站流程。 1：用户名 密码 验证码 在发起登录请求时要携带发送到服务端 2：如果登陆不成功，首先考虑data数据中是否有动态变化的请求参数（通常情况下动态变化的请求参数都会被隐藏在前台页面源码中） 3：如果携带动态数据登录还是失败，则需要考虑Cookie情况。可以使用Session对象发起网络请求。
线程池提高爬虫效率
客户端代码
# 使用了线程池的爬虫代码 from multiprocessing.dummy import Pool import time start = time.time() urls = [ 'http://127.0.0.1:5000/bobo', 'http://127.0.0.1:5000/jay' ] def get_request(url): page_text = requests.get(url).text print(page_text) pool = Pool(3) pool.map(get_request,urls) print('总耗时：',time.time()-start)
服务器端代码
from flask import Flask import time app = Flask(__name__) @app.route('/bobo') def index_bobo(): time.sleep(2) return 'Hello bobo' @app.route('/jay') def index_jay(): time.sleep(2) return 'Hello jay' @app.route('/tom') def index_tom(): time.sleep(2) return 'Hello tom' if __name__ == '__main__': app.run(threaded=True)
运行后发现原本需要4秒完成的任务，使用了线程池之后2秒就完成了。
单线程+多任务异步协程提高爬虫效率
### 单线程+多任务异步协程 - 协程 - 在函数（特殊的函数）定义的时候，如果使用了async修饰的话，则改函数调用后会返回一个协程对象，并且函数内部的实现语句不会被立即执行 - 任务对象 - 任务对象就是对协程对象的进一步封装。任务对象==高级的协程对象==特殊的函数 - 任务对象时必须要注册到事件循环对象中 - 给任务对象绑定回调：爬虫的数据解析中 - 事件循环 - 当做是一个容器，容器中必须存放任务对象。 - 当启动事件循环对象后，则事件循环对象会对其内部存储任务对象进行异步的执行。 - aiohttp:支持异步网络请求的模块
简单了解几个概念
协程
import asyncio def callback(task):#作为任务对象的回调函数 print('i am callback and ',task.result()) async def test(): print('i am test()') return 'bobo' c = test() #封装了一个任务对象 task = asyncio.ensure_future(c) task.add_done_callback(callback) #创建一个事件循环的对象 loop = asyncio.get_event_loop() loop.run_until_complete(task)
多任务
import asyncio import time start = time.time() #在特殊函数内部的实现中不可以出现不支持异步的模块代码 async def get_request(url): await asyncio.sleep(2) print('下载成功:',url) urls = [ 'www.1.com', 'www.2.com' ] tasks = [] for url in urls: c = get_request(url) # 创建任务 task = asyncio.ensure_future(c) tasks.append(task) loop = asyncio.get_event_loop() #注意：挂起操作需要手动处理 loop.run_until_complete(asyncio.wait(tasks)) print(time.time()-start)
示例应用
import requests import aiohttp import time import asyncio s = time.time() urls = [ 'http://127.0.0.1:5000/bobo', 'http://127.0.0.1:5000/jay' ] # async def get_request(url): # page_text = requests.get(url).text # return page_text async def get_request(url): async with aiohttp.ClientSession() as s: async with await s.get(url=url) as response: page_text = await response.text() print(page_text) return page_text tasks = [] for url in urls: c = get_request(url) task = asyncio.ensure_future(c) tasks.append(task) loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.wait(tasks)) print(time.time()-s)
单线程+多任务异步协程实例
import aiohttp import asyncio import time from lxml import etree start = time.time() urls = [ 'http://127.0.0.1:5000/bobo', 'http://127.0.0.1:5000/jay', 'http://127.0.0.1:5000/tom', 'http://127.0.0.1:5000/bobo', 'http://127.0.0.1:5000/jay', 'http://127.0.0.1:5000/tom', 'http://127.0.0.1:5000/bobo', 'http://127.0.0.1:5000/jay', 'http://127.0.0.1:5000/tom', 'http://127.0.0.1:5000/bobo', 'http://127.0.0.1:5000/jay', 'http://127.0.0.1:5000/tom' ] #特殊的函数：请求发送和响应数据的捕获 #细节：在每一个with前加上async，在每一个阻塞操作的前边加上await async def get_request(url): async with aiohttp.ClientSession() as s: #s.get(url,headers,proxy="http://ip:port",params) async with await s.get(url) as response: page_text = await response.text()#read()返回的是byte类型的数据 return page_text #回调函数 def parse(task): page_text = task.result() tree = etree.HTML(page_text) parse_data = tree.xpath('//li/text()') print(parse_data) tasks = [] for url in urls: c = get_request(url) task = asyncio.ensure_future(c) task.add_done_callback(parse) tasks.append(task) loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.wait(tasks)) print(time.time()-start)
结果发现是可以实现提高效率的效果。