代码:
# _*_ coding:utf-8 _*_ import urllib2 import re #import sys #reload(sys) #sys.setdefaultencoding('utf-8') class Tool: removeImg = re.compile(r'<p class="f_center".*?</p>') removeAddr = re.compile(r'<a.*?>|</a>') replaceLine = re.compile(r'<tr>|<div>|</div>|</p>') replaceTD = re.compile(r'<td>') replacePara = re.compile(r'<p.*?>') replaceBR = re.compile(r'<br<br>|<br>') removeExtraTag = re.compile(r'<.*?>') def replace(self,text): text = re.sub(self.removeImg,"",text) text = re.sub(self.removeAddr,"",text) text = re.sub(self.replaceLine," ",text) text = re.sub(self.replaceTD," ",text) text = re.sub(self.replacePara," "+" ",text) text = re.sub(self.replaceBR," ",text) text = re.sub(self.removeExtraTag,"",text) return text.strip() class WYXW: def __init__(self,baseUrl): self.baseURL = baseUrl self.user_agent = 'Mozilla/4.0 (compatible;MSIE 5.5; Windows NT)' self.headers = {'User-Agent':self.user_agent} #self.file = None self.fileName = u'网易新闻' self.tool = Tool() def get_homepage(self): url = self.baseURL request = urllib2.Request(url,headers = self.headers) response = urllib2.urlopen(request) content = response.read().decode('utf-8','ignore') #print content#.encode('gbk','ignore') return content def extract_url(self,homepage): pattern = "http://news.163.com/d{2}/d{4}/d{2}/w{16}.html" news_url = re.findall(pattern,homepage) #print news_url return news_url def extract_sub_web_time(self,sub_web): pattern = re.compile(r'd{4}-d{2}-d{2} d{2}:d{2}:d{2}',re.S) time = re.findall(pattern,sub_web) print time[0] return time[0] def extract_sub_web_source(self,sub_web): pattern = re.compile(r'<a id="ne_article_source".*?>(.*?)</a>') source = re.findall(pattern,sub_web) print source[0] return source[0] def extract_sub_web_title(self,sub_web): #pattern = "<title>.+</title>" #pattern = '<h1 id="h1title" class="ep-h1">(.*?)</h1>' pattern = re.compile(r'<h1 id="h1title" class="ep-h1">(.*?)</h1>',re.S) title = re.findall(pattern,sub_web) if title is not None: print title[0] return title[0] else: return None def extract_sub_web_content(self,sub_web): #pattern = "<div id="Cnt-Main-Article-QQ".*</div>" pattern = re.compile(r'<div id="endText".*?>(.*?)<!.*?-->',re.S) content = re.findall(pattern,sub_web) #print content[0] if content is not None: return content[0] else: return None def writeData(self,fName): if fName is not None: file = open(fName + '.txt',"w+") else: file = open(self.fileName + '.txt',"w+") homepage = self.get_homepage() news_urls = self.extract_url(homepage) for url in news_urls: print url web = urllib2.urlopen(url).read() title = self.extract_sub_web_title(web).strip() content = self.extract_sub_web_content(web) time = self.extract_sub_web_time(web).strip() source = self.extract_sub_web_source(web).strip() if content is not None: content = self.tool.replace(content) news = title + " " + time + " " + source + " " + content + " " file.write(news) sep = " " + "-------------------------------------------------------------------------" + " " file.write(sep) print u"新闻写入成功" + " " baseUrl = "http://news.163.com" wyxw = WYXW(baseUrl) wyxw.writeData(None)