实现思路：
由于反扒机制，所以需要做代理切换，去爬取，内容通过BeautifulSoup去解析，最后入mysql库
1.在西刺免费代理网获取代理ip，并自我检测是否可用
2.根据获取的可用代理ip去发送requests模块的请求，带上代理
3.内容入库
注：日志模块在上一篇随笔
下面附上代码
1.可用代理获取
# -*- coding: utf-8 -*- import random import time import requests from bs4 import BeautifulSoup import log_config logger = log_config.getlogger('ip_pool', 'ip_pool.log') class IPProxyPool: # 初始化，定义一个空数组ip_list用于存储ip代理 def __init__(self): # 代理ip获取网址 self.proxy_url_list = ['http://www.xicidaili.com', 'http://www.xicidaili.com/nn', 'http://www.xicidaili.com/nn/2'] self.ip_list = [] self.headers = {'Host': 'www.xicidaili.com', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36' } self.user_agent_list = [ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1", "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24" ] def get_xici_all_ip(self): ip_lists = [] for proxy_url in self.proxy_url_list: html = requests.get(proxy_url, headers=self.headers) content = html.text soup = BeautifulSoup(content, "lxml") ip_body = soup.find("table", attrs={"id": "ip_list"}) ip_page_lists = ip_body.find_all("tr", attrs={"class": "odd"}) ip_lists = ip_lists + ip_page_lists return ip_lists def get_ip_list(self): ip_lists = self.get_xici_all_ip() ip_test_pool = [] for ip in ip_lists: http_type = ip.find_all("td")[5].get_text() if http_type == 'HTTP': ip_test_account = ip.find_all("td")[1].get_text() ip_test_port = ip.find_all("td")[2].get_text() ip_port_dict = {ip_test_account: ip_test_port} ip_test_pool.append(ip_port_dict) for ipn in ip_test_pool: ip_addr = "http://" for ip, port in ipn.items(): ip_addr = ip_addr + ip + ':' + port # ip代理有效性检验 statu = self.check_ip(ip_addr) if statu: # 将有效ip代理存储至数组ip_list中 self.ip_list.append(ip_addr.strip()) def check_ip(self, ip): return self.microbell_proxy_ip(ip) def microbell_proxy_ip(self, ip): try: test_url = 'http://www.microbell.com/elitelist.html' proxy = {'http': ip} user_agent = self.random_agent() headers_agent = {'User-Agent': user_agent} response_body = requests.get(test_url, headers=headers_agent, proxies=proxy, timeout=5) if response_body.status_code == 200: # 即使返回了200,也可能不是我们访问的页面，而是代理给我们的页面,所以还需要做判断 #response_body.encoding('gbk') content = response_body.text soup = BeautifulSoup(content, "lxml") body = soup.find("div", attrs={"class": "index_docmain"}) if body is None: return False if body.get_text() != "": logger.info("ok proxy ip %s" % ip) return True else: return False else: return False except Exception as e: logger.exception(e.message) time.sleep(1) return False def random_agent(self): user_agent = random.choice(self.user_agent_list) return user_agent if __name__ == "__main__": IPProxyPool = IPProxyPool() IPProxyPool.get_ip_list() print IPProxyPool.ip_list # proxies = { # "http": "http://118.190.95.35:9001" # 代理ip # } # # headers = { # 'Host': 'www.4399.com', # 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36' # # } # # http_url = "http://www.4399.com/" # try: # res = requests.get(url=http_url, headers=headers, proxies=proxies, timeout=3) # if res.status_code == 200: # print u"访问网页成功" # else: # print "faile" # except Exception as e: # print e
2.解析界面，获取想要的内容，并入库
#!/usr/bin/env python # -*- coding: utf-8 -*- import ConfigParser import datetime import sys import pymysql import requests from bs4 import BeautifulSoup import log_config import time import random from agent_and_proxy_ip_pool import IPProxyPool logger = log_config.getlogger('report', 'report.log') # 是否获取今天的数据 0 表示获取今天的数据，1表示获取全部数据 2表示页面没有数据 get_today_data = 1 if len(sys.argv) != 1: if sys.argv[1] == 1: get_today_data = 1 else: print 'input error,please input 0->today ,1->all data' exit() class research_report: def __init__(self): conf = ConfigParser.ConfigParser() conf.read("mysql.conf") self.ip_proxy_pool = IPProxyPool() # self.ip_proxy_pool.get_ip_list() # self.ip_pool = self.ip_proxy_pool.ip_list # logger.info('You can currently use IP %s' % self.ip_pool)
#多个可用的cookies
self.cookies_pool = [ 'c=; ASPSESSIONIDQCQRQQCR=LEOOBOJCBAMFFDHMFBHFJKEE; __guid=188006958.3779224451650617000.1539657585525.2588; ASPSESSIONIDSATRTTDQ=MCDEIPFDLLKBNHPBBEMGBGFC; safedog-flow-item=C07B93F771; UM_distinctid=16680b1e9e411f-0674a4c85ccc2-454c092b-1fa400-16680b1e9e539d; CNZZDATA1752123=cnzz_eid%3D2075545357-1539752826-%26ntime%3D1539752826; Hm_lvt_d554f0f6d738d9e505c72769d450253d=1539757436; robih=vXuWjYMDvV6XmNxOuNmP; MBpermission=0; MBname=sunyue1993; did=67A671BFE; monitor_count=6; Hm_lpvt_d554f0f6d738d9e505c72769d450253d=1539757719' ] self.get_today = get_today_data self.user = conf.get("mysql", "user") self.mysql_password = conf.get("mysql", "password") self.database_name = conf.get("mysql", "database") self.host = conf.get("mysql", "host") self.port = conf.get("mysql", "port") self.site_url = 'http://www.microbell.com/' self.page_url = 'http://www.microbell.com/elitelist_1_0.html' self.headers = {'Host': 'www.microbell.com', 'Accept': 'application/json, text/javascript, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate, sdch', 'Accept-Language': 'zh-CN,zh;q=0.8', 'Connection': 'keep-alive' } # 生成随机agent def get_random_headers(self): # self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36' self.headers['User-Agent'] = self.ip_proxy_pool.random_agent() self.headers['Cookie'] = random.choice(self.cookies_pool) # 获取指定数组中的随机ip def get_random_proxy(self): proxy_ip = random.choice(self.ip_pool) proxies = {'http': proxy_ip} return proxies # 获取列表页面 def get_html_content(self, page_num_url): try: self.get_random_headers() req = requests.get(page_num_url, headers=self.headers, timeout=5) req.encoding = 'gbk' text = req.text soup = BeautifulSoup(text, "lxml") # soup = body.prettify #美化 report_list = soup.find_all("div", attrs={"class": "classbaogao_sousuo_list"}) list_data = [] logger.info("%s owner %s pages" % (page_num_url, len(report_list))) if len(report_list) == 0: return 2 for report_item in report_list: url = self.site_url + report_item.table.tr.find_all("td")[1].a["href"] title = report_item.table.tr.find_all("td")[1].a["title"] item_data = {"url": url, "title": title} list_data.append(item_data) end_flag = self.get_list_page_data(list_data) return end_flag except Exception as e: logger.exception("get list %s page fail error info : %s" % (page_num_url, e)) return 2 # 获取一页数据38条的每条的详情 def get_list_page_data(self, list_data): try: # 一页数据(38)数组入库一次 page_datas = [] now_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.get_random_headers() #proxy_ip = self.get_random_proxy() for item_data in list_data: retry_num = 0 while retry_num < 3: try: # 休眠两秒，避免被封 t = random.uniform(3, 5) time.sleep(t) req = requests.get(item_data["url"], headers=self.headers, timeout=5) req.encoding = 'gbk' text = req.text soup = BeautifulSoup(text, "lxml") detail_div = soup.find("div", attrs={"class": "leftn2"}) tr_s = detail_div.table.find_all("tr") public_time = tr_s[0].find_all("td")[2].span.get_text() if self.get_today == 0: # 如果是爬取今天的数据，会进入这个判断，如果当期获取的文件的时间早于今天就直接退出循环 logger.info("now spider today data") today = datetime.date.today() today_time = int(time.mktime(today.timetuple())) time_array = time.strptime(public_time, "%Y-%m-%d %H:%M:%S") pub_time = int(time.mktime(time_array)) if pub_time < today_time: break abstract_br_replace = soup.find("div", attrs={"class": "p_main"}).p.span str1 = str(abstract_br_replace).replace("<br/>", r" ") abstract_object = BeautifulSoup(str1, "lxml") [s.extract() for s in abstract_object("font")] abstract = abstract_object.get_text() sec_name = tr_s[0].find_all("td")[0].span.get_text() sec_code = tr_s[0].find_all("td")[1].span.get_text() report_type = tr_s[1].find_all("td")[0].span.get_text() doc_type = tr_s[1].find_all("td")[1].span.get_text() author = tr_s[1].find_all("td")[2].span.get_text() provenance = tr_s[2].find_all("td")[0].span.get_text() pages = tr_s[2].find_all("td")[1].span.get_text() rec_rate = tr_s[2].find_all("td")[2].span.get_text() doc_size = tr_s[3].find_all("td")[0].span.get_text() promulgator = tr_s[3].find_all("td")[1].span.get_text() #doc_url_str = soup.find("div", attrs={"class": "anniu_main"}).a["onclick"] doc_url_str = "" doc_url_list = doc_url_str.split(",") doc_url = self.site_url + doc_url_list[2] title = item_data["title"] create_time = now_date update_time = now_date page_data = [title, sec_name, sec_code, public_time, report_type, doc_type, author, provenance, pages, rec_rate, doc_size, doc_url, promulgator, abstract, create_time, update_time] page_datas.append(page_data) break except Exception as e: retry_num += 1 if retry_num == 3: logger.warning("current page is not get %s" % item_data) if len(page_datas) > 0: self.set_data_mysql(page_datas) if self.get_today == 0: if len(page_datas) < 38: return 0 return 1 else: return 2 except Exception as e: logger.error("get detail page fail" % list_data, e) return 2 # 批量插入mysql def set_data_mysql(self, page_datas): # 创建连接 conn = pymysql.connect(host=self.host, port=int(self.port), user=self.user, passwd=self.mysql_password, db=self.database_name) try: # 创建游标 cursor = conn.cursor() sql = "INSERT INTO report(title,sec_name,sec_code,public_time,report_type,doc_type,author," "provenance,pages,rec_rate,doc_size,doc_url,promulgator,abstract,create_time,update_time) " "VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" effect_row = cursor.executemany(sql, page_datas) # 提交sql，不提交不会进入mysql conn.commit() logger.info("already into dabatabase %s" % effect_row) finally: conn.close() # 登录，获取cookie,暂时没用 # def login_in(self): # data = { # 'namelogin': self.user_name, # 'pwdlogin': self.password # } # req = requests.post(self.login_url, headers=self.headers, data=data) # req.encoding = req.apparent_encoding # cookies = req.cookies.get_dict() # print cookies # http://www.microbell.com/elitelist_1_0.html 初始页面，后面的页面只有url中的"1"这个值会变动 def process(self): # 分析页面，总共不超过360页 if get_today_data == 0: for i in range(1, 20): base_url = "http://www.microbell.com/elitelist_%s_0.html" % i logger.info("当前获取页面url=%s" % base_url) end_flag = self.get_html_content(base_url) if end_flag == 0: logger.info("The page %s is already the last page" % base_url) break else: for i in reversed(range(1, 107)): base_url = "http://www.microbell.com/elitelist_%s_0.html" % i logger.info("当前获取页面url=%s" % base_url) self.get_html_content(base_url) if __name__ == "__main__": research_class = research_report() research_class.process() else: research_class = research_report() research_class.process()