上次纠结了那么久。结果不用编码也是可以匹配邮箱的。
下面是一个用队列实现,广度优先的简单爬虫代码。先就这样吧,目测暂时不会再理它了,以后有时间再修改。(又是一个烂尾。。。。。)
View Code
1 # -*- coding: cp936 -*- 2 import urllib2 3 import re 4 from pyquery import PyQuery as pq 5 from lxml import etree 6 7 #mailpattern = re.compile('[^\._:>\\-][\w\.-]+@(?:[A-Za-z0-9]+\.)+[A-Za-z]+') 8 mailpattern = re.compile('[A-Za-z0-9_]+@(?:[A-Za-z0-9]+\.)+[A-Za-z]+') 9 10 htmlcount = 0 #to count the urls 11 maxcount = 3000 # the max count 12 allUrls = set() 13 allMails = set() 14 UrlsQlist = [] 15 UrlsQdict = {} 16 url = "http://www.163.com" 17 fmails = open("E:/py/crawler/mailresult.txt","a") 18 furls = open("E:/py/crawler/urlresult.txt","a") 19 20 21 22 23 def geturls(data):#the function to get the urls in the html 24 urls = set() 25 if data: 26 d = pq(data) 27 label_a = d.find('a')#用pyquery库去找到 a 标签. 28 if label_a: 29 label_a_href = d('a').map(lambda i,e:pq(e)('a').attr('href')) 30 for u in label_a_href: 31 if u[0:10]!="javascript" : 32 if u[0:4] == "http": 33 urls.add(u) 34 else: 35 urls.add(url + u) 36 #for u in urls: 37 #print u 38 return urls 39 else: 40 return None 41 42 def gethtml(url): 43 try: 44 fp = urllib2.urlopen(url) 45 except: 46 print "urllib2.urlopen error" 47 return None 48 else: 49 mybytes =fp.read() 50 fp.close() 51 return mybytes 52 53 def savemails(data): # the function to save the emails 54 if data: 55 mailResult = mailpattern.findall(data) 56 mailResultset = set(mailResult) 57 if mailResultset: 58 allMails.update(mailResultset) 59 60 def savehtml(pagecontent,count): 61 if pagecontent != None: 62 f = open("E:/py/crawler/html/"+str(count)+".html","w") 63 f.write(pagecontent) 64 f.close() 65 else: 66 f = open("E:/py/crawler/html/"+str(count)+"error"+".html","w") 67 f.write("this page empty") 68 f.close() 69 70 def BFS(firstUrl): 71 global htmlcount 72 global maxcount 73 allUrls.add(firstUrl) 74 UrlsQlist = list(allUrls) 75 while htmlcount < maxcount : #数量小于最大值 76 tempUrl = UrlsQlist.pop(0)# the queue 77 myWebStr = gethtml(tempUrl) 78 savehtml(myWebStr,htmlcount) 79 savemails(myWebStr) 80 firstUrls_set = geturls(myWebStr)#初始页面的处理 81 if firstUrls_set != None: 82 allUrls.update(firstUrls_set) #记录全部 url 83 for u in firstUrls_set: 84 if u not in UrlsQlist: 85 UrlsQlist.append(u) 86 htmlcount = htmlcount + 1 87 88 89 BFS(url) 90 for u in allMails: 91 try: 92 fmails.write(u) 93 fmails.write('\n') 94 except: 95 continue 96 for u in allUrls: 97 try: 98 furls.write(u) 99 furls.write('\n') 100 except: 101 continue 102 fmails.close() 103 furls.close()
2013.5.13 update
本来想在加个多线程。。。。结果看了 好多资料 无处下手,再研究研究 ,日后再改
加了点 url规范化。代码整理如下:
1 import urllib2 2 import re 3 from pyquery import PyQuery as pq 4 from lxml import etree 5 import urlparse 6 import time 7 8 allUrls = set() 9 allMails = set() 10 urlsDownlist = [] 11 12 class mailCrawler: 13 def __init__(self,mailExpression,start_url,maxcount): 14 ''' mailExpressoin 邮箱的正则表达式; 15 start_url开始邮箱; 16 maxcount最大数量''' 17 self.mailpattern = re.compile(mailExpression) 18 self.maxcount = maxcount 19 self.htmlcount = 0 20 self.UrlsQlist = []#url queue 实现广度优先 21 self.url = start_url 22 23 24 def url_normal(self,url): 25 '''url 规范化 ''' 26 scheme,netloc,path,query = urlparse.urlsplit(url)[:4] 27 netloc = netloc.lower() 28 29 url.encode("utf-8") 30 31 if path: 32 path = re.sub('/{2,}','/',path)#去除url中的重复/ 33 path = re.sub(r'\.$','',path)#去除url中结尾多余的点 34 path = re.sub('/$','',path)#去除url中结尾多余的/ 35 path = re.sub('\s','',path)#取出url中的空格 36 if query: 37 return '%s://%s%s?%s' % (scheme,netloc,path or '/',query) 38 else: 39 return '%s://%s%s' % (scheme,netloc,path) 40 41 def geturls(self,data): 42 '''解析html中的url''' 43 urls = set() 44 if data: 45 d = pq(data) 46 label_a = d.find('a')#用pyquery库去找到 a 标签. 47 if label_a: 48 label_a_href = d('a').map(lambda i,e:pq(e)('a').attr('href')) 49 for u in label_a_href: 50 if u[0:10]!="javascript" and u[0:6]!="mailto" : 51 if u[0:4] == "http": 52 normal_url = self.url_normal(u) 53 urls.add(normal_url) 54 else: 55 normal_url = self.url_normal(self.url + u) 56 urls.add(normal_url) 57 return urls 58 else: 59 return None 60 61 def gethtml(self,url): 62 '''下载html 5s超时''' 63 try: 64 fp = urllib2.urlopen(url,None,5) 65 except: 66 print "urllib2.urlopen error or timeout" 67 return None 68 else: 69 mybytes =fp.read() 70 fp.close() 71 return mybytes 72 73 def savemails(self,data): 74 '''将抓取到的url存放到 allmails中 ,set去重复''' 75 global allMails 76 if data: 77 mailResult = self.mailpattern.findall(data) 78 mailResultset = set(mailResult) 79 if mailResultset: 80 allMails.update(mailResultset) 81 82 def savehtml(self,pagecontent,htmlcount,url): 83 '''保存html文件 ''' 84 if pagecontent != None: 85 f = open("E:/py/crawler/html/"+str(htmlcount)+".html","w") 86 f.write(pagecontent) 87 f.close() 88 else: 89 f = open("E:/py/crawler/html/"+str(htmlcount)+"error"+".html","w") 90 try: 91 f.write(url) 92 except: 93 f.write("encode error") 94 f.close() 95 96 def BFS(self): 97 '''用队列实现广度优先,爬取url ''' 98 global allUrls 99 global urlsDownlist 100 allUrls.add(self.url) 101 self.UrlsQlist = list(allUrls) 102 while self.htmlcount < self.maxcount : #数量小于最大值 103 tempUrl = self.UrlsQlist.pop(0)# the queue 104 print tempUrl 105 urlsDownlist.append(tempUrl) 106 myWebStr = self.gethtml(tempUrl) 107 self.savehtml(myWebStr,self.htmlcount,tempUrl) 108 self.savemails(myWebStr) 109 firstUrls_set = self.geturls(myWebStr)#初始页面的处理 110 if firstUrls_set != None: 111 for u in firstUrls_set: 112 if u not in allUrls: 113 allUrls.add(u) 114 self.UrlsQlist.append(u) 115 self.htmlcount = self.htmlcount + 1 116 117 118 def main(): 119 reg = r'[A-Za-z0-9_]+@(?:[A-Za-z0-9]+\.)+[A-Za-z]+' 120 url = "http://www.baidu.com" 121 count = 100 122 fmails = open("E:/py/crawler/mailresult.txt","a") 123 furls = open("E:/py/crawler/urlresult.txt","a") 124 fdownUrls = open("E:/py/crawler/urlDownresult.txt","a") 125 newcrawler = mailCrawler(reg,url,count) 126 newcrawler.BFS() 127 for u in allMails: 128 try: 129 fmails.write(u) 130 fmails.write('\n') 131 except: 132 continue 133 for u in allUrls: 134 try: 135 furls.write(u) 136 furls.write('\n') 137 except: 138 continue 139 for u in urlsDownlist: 140 try: 141 fdownUrls.write(u) 142 fdownUrls.write('\n') 143 except: 144 continue 145 fmails.close() 146 furls.close() 147 fdownUrls.close() 148 149 if __name__ == '__main__': 150 main()