# 嗯，。。。因为经常需要使用代理去抓一点东西，就有了下面一段代码，第一版不是很好，后面用到了再来优化
import re,pymysql,time,redis from urllib.request import Request from urllib.request import urlopen headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36', } url = 'http://www.xicidaili.com/wt/' db = pymysql.connect(host='127.0.0.1', user='root', password="liu", database='test', port=3306, charset='utf8') cur = db.cursor() def url_response(url,cur): response = urlopen(Request(url,headers=headers)).read() response = response.decode() pattern='<td>(.*?)</td>s+<td>(d+)</td>s+<td>s+<a href="/.*?">[u4e00-u9fa5]+</a>s+</td>s+<td class="country">高匿</td>s+<td>(w+)</td>s+<td class="country">s+<div title="(d.d+)秒"' regex = re.compile(pattern) ip_list = regex.findall(response) for i in ip_list: out_time = float(i[3]) ip_ = i[0]+':'+i[1] sql = 'select ip_ from ip where ip_ = "%s"'%ip_ cur.execute(sql) if cur.fetchone(): print('重复数据跳过') continue if out_time < 1: sql = "insert into ip(ip_,time_,xy_) values('%s','%s','%s')"%(ip_,out_time,i[2]) cur.execute(sql) print('插入成功，',i) else: pass for i in range(1,3): _ = url+str(i) url_response(_,cur) db.commit() time.sleep(2)
# 第一版使用 mysql存着也没啥用处 ，然后就给来了redis # 加入线程池的使用，让抓取更加速度 import re,time,redis from concurrent.futures import ThreadPoolExecutor from urllib.request import Request from urllib.request import urlopen headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36', } url = 'http://www.xicidaili.com/wt/' class R(object): def __init__(self): r_pool =redis.ConnectionPool(host='127.0.0.1',db=0,password=None, port=6379) self.redis_obj = redis.Redis(connection_pool=r_pool) def setex(self,name,value,time): return self.redis_obj.setex(name,value,time) def get(self,name): return self.redis_obj.get(name) def url_response(url,redis_obj): response = urlopen(Request(url,headers=headers)).read() response = response.decode() pattern='<td>(.*?)</td>s+<td>(d+)</td>s+<td>s+<a href="/.*?">[u4e00-u9fa5]+</a>s+</td>s+<td class="country">高匿</td>s+<td>(w+)</td>s+<td class="country">s+<div title="(d.d+)秒"' regex = re.compile(pattern) ip_list = regex.findall(response) for i in ip_list: out_time = float(i[3]) ip_ = i[0]+':'+i[1] if redis_obj.get(ip_): print('重复数据跳过') continue if out_time < 1: redis_obj.setex(ip_,1,60*30) print('插入成功，',ip_) else: pass r = R() T = ThreadPoolExecutor(4) for i in range(1,5): _ = url+str(i) T.submit(url_response,_,r) print('执行完成 ') T.shutdown()
2018-12-17:
第二版出错更新： 15行 ：return self.redis_obj.setex(name,time,value) #此处已改正
# python 3.7 from lxml import etree import requests,time,redis class Kuai_IP(object): def __init__(self): self.headers = { 'Host': 'www.kuaidaili.com', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36', 'Referer': 'https://www.kuaidaili.com/free/inha/1/' } self.static='https://www.kuaidaili.com/free/inha/%s/' Con_pool = redis.ConnectionPool(host='127.0.0.1') self.r = redis.Redis(connection_pool=Con_pool) def getPage(self,page_index): if page_index==1: self.headers['Referer']='https://www.kuaidaili.com/free/inha/' else: self.headers['Referer']='https://www.kuaidaili.com/free/inha/'+str(page_index-1)+'/' res = requests.get(url=self.static%page_index,headers=self.headers) self.parse(res.text) def parse(self,res): html = etree.HTML(res) r_list = html.xpath('//tbody/tr/td/text()') if r_list: for i in range(1, int(len(r_list) / 7)): _ =r_list[i * 7:(i + 1) * 7] self.r.setex(_[3]+'://'+_[0]+':'+_[1],30,_[3]) print(_) else: print(r_list) def work_on(self): page_index = 2 # 爬取页数 for i in range(1,page_index+1): self.getPage(i) print(i, '---------') time.sleep(2) if __name__ == '__main__': ip = Kuai_IP() ip.work_on()
2018-12-20
嗯，这次由于用到的代理比较多，就把西刺和快代理的代码合到了一起，没做什么大的改进，
1 代理ip格式全部成为 requests代理的形式{'http':'xxx://xx.xx.xx.xx:xxx'} 方便requests的调用
# -*- coding:utf-8 -*- # @time:2018-12-20 22:23 import re,redis,time,requests from concurrent.futures import ThreadPoolExecutor from urllib.request import Request from urllib.request import urlopen page = 10 #定义抓取页数，由于快代理一页的ip比较少，然后我就在此的基础上+10 headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36', } url = 'http://www.xicidaili.com/wt/' class R(object): def __init__(self): r_pool =redis.ConnectionPool(host='127.0.0.1',db=0,password=None, port=6379) self.redis_obj = redis.Redis(connection_pool=r_pool) def setex(self,name,value,time): return self.redis_obj.setex(name, value, time) def get(self,name): return self.redis_obj.get(name) def url_response(url,redis_obj): response = urlopen(Request(url,headers=headers)).read() response = response.decode() pattern='<td>(.*?)</td>s+<td>(d+)</td>s+<td>s+<a href="/.*?">[u4e00-u9fa5]+</a>s+</td>s+<td class="country">高匿</td>s+<td>(w+)</td>s+<td class="country">s+<div title="(d.d+)秒"' regex = re.compile(pattern) ip_list = regex.findall(response) for i in ip_list: out_time = float(i[3]) ip_ = i[0]+':'+i[1] if redis_obj.get(ip_): print('重复数据跳过') continue if out_time < 1: ip_ = "HTTP://"+str(ip_) redis_obj.setex(ip_,1,60*30*20) print('插入成功，',ip_) else: pass from lxml import etree class Kuai_IP(object): def __init__(self): self.headers = { 'Host': 'www.kuaidaili.com', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36', 'Referer': 'https://www.kuaidaili.com/free/inha/1/' } self.static='https://www.kuaidaili.com/free/inha/%s/' Con_pool = redis.ConnectionPool(host='127.0.0.1',port=6379,db=0) self.r = redis.Redis(connection_pool=Con_pool) def getPage(self,page_index): if page_index==1: self.headers['Referer']='https://www.kuaidaili.com/free/inha/' else: self.headers['Referer']='https://www.kuaidaili.com/free/inha/'+str(page_index-1)+'/' res = requests.get(url=self.static%page_index,headers=self.headers) self.parse(res.text) def parse(self,res): html = etree.HTML(res) r_list = html.xpath('//tbody/tr/td/text()') if r_list: for i in range(1, int(len(r_list) / 7)): _ =r_list[i * 7:(i + 1) * 7] self.r.setex(_[3]+'://'+_[0]+':'+_[1],_[3],30*60*24) print(_) else: print(r_list) def work_on(self): page_index = page+10 # 爬取页数 for i in range(1,page_index+1): self.getPage(i) print(i, '---------') time.sleep(2) if __name__ == '__main__': r = R() T = ThreadPoolExecutor(4) for i in range(1,page): _ = url+str(i) T.submit(url_response,_,r) print('执行完成 ') T.shutdown() ip = Kuai_IP() ip.work_on()
2018-12-24
嗯，上面代理在80行出现错误，因为快代理的这个超时时间有时出现的是数字+文字，然后在设置缓存的时候就
出现了【 value is not an integer or out of range】
# -*- coding:utf-8 -*- # @time:2018-12-18 22:23 # @Auther:1043453579@qq.com import re,redis,time,requests from concurrent.futures import ThreadPoolExecutor from urllib.request import Request from urllib.request import urlopen page = 10 headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36', } url = 'http://www.xicidaili.com/wt/' class R(object): def __init__(self): r_pool =redis.ConnectionPool(host='127.0.0.1',db=0,password=None, port=6379) self.redis_obj = redis.Redis(connection_pool=r_pool) def setex(self,name,value,time): return self.redis_obj.setex(name, time, value) def get(self,name): return self.redis_obj.get(name) def url_response(url,redis_obj): response = urlopen(Request(url,headers=headers)).read() response = response.decode() pattern='<td>(.*?)</td>s+<td>(d+)</td>s+<td>s+<a href="/.*?">[u4e00-u9fa5]+</a>s+</td>s+<td class="country">高匿</td>s+<td>(w+)</td>s+<td class="country">s+<div title="(d.d+)秒"' regex = re.compile(pattern) ip_list = regex.findall(response) for i in ip_list: out_time = float(i[3]) ip_ = i[0]+':'+i[1] if redis_obj.get(ip_): print('重复数据跳过') continue if out_time < 1: ip_ = "HTTP://"+str(ip_) redis_obj.setex(ip_,1,60*30*20) print('插入成功，',ip_) else: pass r = R() T = ThreadPoolExecutor(4) for i in range(1,page): _ = url+str(i) T.submit(url_response,_,r) print('执行完成 ') T.shutdown() from lxml import etree class Kuai_IP(object): def __init__(self): self.headers = { 'Host': 'www.kuaidaili.com', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36', 'Referer': 'https://www.kuaidaili.com/free/inha/1/' } self.static='https://www.kuaidaili.com/free/inha/%s/' Con_pool = redis.ConnectionPool(host='127.0.0.1',port=6379,db=0) self.r = redis.Redis(connection_pool=Con_pool) def getPage(self,page_index): if page_index==1: self.headers['Referer']='https://www.kuaidaili.com/free/inha/' else: self.headers['Referer']='https://www.kuaidaili.com/free/inha/'+str(page_index-1)+'/' res = requests.get(url=self.static%page_index,headers=self.headers) self.parse(res.text) def parse(self,res): html = etree.HTML(res) r_list = html.xpath('//tbody/tr/td/text()') if r_list: for i in range(1, int(len(r_list) / 7)): _ =r_list[i * 7:(i + 1) * 7] print('1',_) #exit() self.r.setex(_[3]+'://'+_[0]+':'+_[1],30*60*24,30*60*24) # print(_) else: print(r_list) def work_on(self): page_index = page+10 # 爬取页数 for i in range(1,page_index+1): self.getPage(i) print(i, '---------') time.sleep(2) ip = Kuai_IP() ip.work_on()