4.2 爬虫的编写
爬虫的思路我们上面已经讲过了，先完成url的管理，我们单独将他作为一个类 文件保存在lib/core/UrlManager.py。
#!/usr/bin/env python
#-*- coding:utf-8 -*-
class UrlManager(object):
def __init__(self):
self.new_urls = set()
self.old_urls = set()
def add_new_url(self, url):
if url is None:
return
if url not in self.new_urls and url not in self.old_urls:
self.new_urls.add(url)
def add_new_urls(self, urls):
if urls is None or len(urls) == 0:
return
for url in urls:
self.add_new_url(url)
def has_new_url(self):
return len(self.new_urls) != 0
def get_new_url(self):
new_url = self.new_urls.pop()
self.old_urls.add(new_url)
return new_url
同时为了方便，我们也可以将下载功能单独的作为一个类使用,文件保存在lib/core/Downloader.py简单写一下get/post方法即可。
#!/usr/bin/env python
#-*- coding:utf-8 -*-
import requests
class Downloader(object):
def get(self,url):
r = requests.get(url,timeout=10)
if r.status_code != 200:
return None
_str = r.text
return _str
def post(self,url,data):
r = requests.post(url,data)
_str = r.text
return _str
def download(self, url,htmls):
if url is None:
return None
_str = {}
_str["url"] = url
try:
r = requests.get(url, timeout=10)
if r.status_code != 200:
return None
_str["html"] = r.text
except Exception as e:
return None
htmls.append(_str)
特别说明下，因为我们爬虫会是多线程的，所以类中有个download方法是专门为多线程下载用的。
在 lib/core/Spider.py 创建爬虫。
爬虫代码如下：
#!/usr/bin/env python
#-*- coding:utf-8 -*-
from lib.core import Downloader,UrlManager
import threading
from urlparse import urljoin
from bs4 import BeautifulSoup
class SpiderMain(object):
def __init__(self,root,threadNum):
self.urls = UrlManager.UrlManager()
self.download = Downloader.Downloader()
self.root = root
self.threadNum = threadNum
def _judge(self, domain, url):
if (url.find(domain) != -1):
return True
else:
return False
def _parse(self,page_url,content):
if content is None:
return
soup = BeautifulSoup(content, 'html.parser')
_news = self._get_new_urls(page_url,soup)
return _news
def _get_new_urls(self, page_url,soup):
new_urls = set()
links = soup.find_all('a')
for link in links:
new_url = link.get('href')
new_full_url = urljoin(page_url, new_url)
if(self._judge(self.root,new_full_url)):
new_urls.add(new_full_url)
return new_urls
def craw(self):
self.urls.add_new_url(self.root)
while self.urls.has_new_url():
_content = []
th = []
for i in list(range(self.threadNum)):
if self.urls.has_new_url() is False:
break
new_url = self.urls.get_new_url()
## sql check
print("craw:" + new_url)
t = threading.Thread(target=self.download.download,args=(new_url,_content))
t.start()
th.append(t)
for t in th:
t.join()
for _str in _content:
if _str is None:
continue
new_urls = self._parse(new_url,_str["html"])
self.urls.add_new_urls(new_urls)
爬虫通过调用craw()方法传入一个网址进行爬行，然后采用多线程的方法下载待爬行的网站，下载后的源码用_parse方法调用BeautifulSoup进行解析，之后将解析出的url列表丢入url管理器中，这样循环，最后只要爬完了网页，爬虫就会停止
我们使用了threading库，进行多线程编写，本项目中，可以自定义需要开启的线程数，线程开启后，每个线程会得到一个url进行下载，然后线程会阻塞，阻塞完毕后线程放行，继续运行。