什么是爬虫?
我们可以把互联网比作一张大网,而爬虫(即网络爬虫)便是在网上爬行的蜘蛛。把网的节点比作一个个网页,爬虫爬到这就相当于访问了该页面,获取了其信息。可以把节点间的连线比作网页与网页之间的链接关系,这样蜘蛛通过一个节点后,可以顺着节点连线继续爬行到达下一个节点,即通过一个网页继续获取后续的网页,这样整个网的节点便可以被蜘蛛全部爬行到,网站的数据就可以被抓取下来了。
代码：
1 from urllib import request 2 from collections import deque 3 from pybloom_live import BloomFilter 4 from lxml import etree 5 import hashlib 6 7 class crawel_bfs: 8 request_header={ 9 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 10 'Accept-Encoding': 'br', 11 'Accept-Language': 'zh-CN,zh;q=0.9', 12 'Cache-Control': 'max-age=0', 13 'Connection': 'keep-alive', 14 'Cookie': 'bid=Kn9AT5duD7k; gr_user_id=32e433a7-19f0-4e17-80c4-56b58d7c0056; _vwo_uuid_v2=5985FEE132C29EC9C840D6C5EDD95323|67c2ccc8467fc02a9cce6928e86ea013; ll="118281"; __yadk_uid=I4Ki5RUaEWOUdpVIjJfRYo1MEuaW36hA; __utmv=30149280.16369; viewed="10483489_1115600_2230208_26857712_1569487_1623164_26708119_26677686"; __utma=30149280.965685249.1516632348.1528892392.1530880979.81; __utmc=30149280; __utmz=30149280.1530880979.81.57.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmt=1; __utmb=30149280.1.10.1530880979; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1530880982%2C%22https%3A%2F%2Fwww.douban.com%2F%22%5D; _pk_ses.100001.4cf6=*; __utma=223695111.2038558801.1520348154.1528892435.1530880982.55; __utmb=223695111.0.10.1530880982; __utmc=223695111; __utmz=223695111.1530880982.55.51.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; _pk_id.100001.4cf6=da4243a2a9e242f1.1520348154.54.1530881042.1528892472.', 15 'Host': 'movie.douban.com', 16 'Referer': 'https://www.douban.com/', 17 'Upgrade-Insecure-Requests': '1', 18 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36' 19 } 20 21 cur_level=0 22 max_level=2 23 24 download_bf=BloomFilter(1024*1024,0.01) 25 26 childque=deque() 27 nowque=deque() 28 29 def __init__(self,url,file_md5name,file_urlname): 30 self.file_urlNameMd5_name=file_md5name 31 self.file_urlName_name=file_urlname 32 self.deal_file_md5=open(self.file_urlNameMd5_name,'r') 33 self.file_md5=self.deal_file_md5.readlines() 34 #用于输入现有的文件 35 for url_md5_name in self.file_md5: 36 #-1表示的是换行符,读入时换行符不会占据两个字符 37 self.download_bf.add(url_md5_name[:-1]) 38 self.deal_file_md5.close() 39 self.file_md5=open(self.file_urlNameMd5_name,'a') 40 self.file_url=open(self.file_urlName_name,'a') 41 self.nowque.append(url) 42 43 def indeque(self,url): 44 self.nowque.append(url) 45 46 def outdeque(self): 47 try: 48 url=self.nowque.popleft() 49 return url 50 except Exception: 51 self.cur_level+=1 52 if self.cur_level==self.max_level: 53 return None 54 if len(self.childque)==0: 55 return None 56 self.nowque=self.childque 57 self.childque=deque() 58 return self.nowque.popleft() 59 60 def crawler(self,url): 61 try: 62 #创建一个request对象，封装一个报文对象 63 req=request.Request(url,headers=self.request_header) 64 #发送报文 65 response=request.urlopen(req) 66 html_page=response.read() 67 #按照固定编码解码 68 html=etree.HTML(html_page.lower().decode('utf-8')) 69 url_list=html.xpath('//a/@href') 70 for url in url_list: 71 if url.find('javascript:')!=-1: 72 continue 73 if url.startswith('http://') is False: 74 if url.startswith('/') is True: 75 url='http://movie.douban.com'+url 76 else: 77 continue 78 if url[-1]=='/': 79 url=url[:-1] 80 temp=hashlib.md5(url.encode('utf-8')).hexdigest() 81 if temp not in self.download_bf: 82 self.download_bf.add(url) 83 self.childque.append(url) 84 self.file_md5.write(temp+'\n') 85 self.file_url.write(url+'\n') 86 except Exception: 87 print("出现异常") 88 89 def startcrawler(self): 90 while True: 91 url=self.outdeque() 92 if url!=None: 93 print("现在爬取"+url+"的超链接") 94 self.crawler(url) 95 else: 96 break 97 self.file_md5.close() 98 self.file_url.close() 99 100 crawel=crawel_bfs("https://movie.douban.com/",'urlmd5.txt', 101 'urlname.txt') 102 crawel.startcrawler()