1 __author__ = 'minmin' 2 #coding:utf-8 3 import re,urllib,sgmllib,os 4 5 #根据当前的url获取html 6 def getHtml(url): 7 page = urllib.urlopen(url) 8 html = page.read() 9 page.close() 10 return html 11 12 #根据html获取想要的文章内容 13 def func(str): 14 result= re.findall(r"<p>([^<>]*)</p>",getHtml(url),re.M) 15 artical ='' 16 for j in result: 17 if len(j)<>0: 18 j = j.replace(" ","") 19 j = j.replace("<STRONG>"," ")#去掉<STRONG>,换成" " 20 j = j.replace("</STRONG>"," ")#去掉</STROGN>换成" " 21 temp = re.findall(r"Copyright.*?",j,re.M); 22 if temp == []: 23 artical = artical + j + ' ' 24 return artical 25 26 #html链接的标签是“a”，链接的属性是“href”，也就是要获得html中所有tag=a，attrs=href 值。 27 class URLPaser(sgmllib.SGMLParser): 28 def reset(self): 29 sgmllib.SGMLParser.reset(self) 30 self.urls = [] 31 32 def start_a(self,attrs): 33 href = [v for k,v in attrs if k == 'href'] 34 if href: 35 self.urls.extend(href) 36 37 IParser = URLPaser() 38 socket = urllib.urlopen("http://tech.sina.com.cn/it/")#打开这个网页 39 40 #fout = file('qq_art_urls.txt','w')#要把这个链接写到这个文件中 41 IParser.feed(socket.read())#分析啦 42 43 reg = 'http://tech.sina.com.cn/it/.*'#这个是用来匹配符合条件的链接，使用正则表达式匹配 44 45 pattern = re.compile(reg) 46 os.getcwd()#获得当前文件夹路径 47 os.path.sep#当前系统路径分隔符 48 49 #判断文件是否存在 50 if os.path.exists('news163_it')==False: 51 os.makedirs('news163_it') 52 53 i = 0 54 url2 = [] 55 for url in IParser.urls:#链接都存在urls里 56 if pattern.match(url): 57 if url not in url2: 58 url2.append(url) 59 artical = func(url) 60 if len(artical)<>0: 61 print url 62 print artical 63 i = i + 1 64 f = open("news163_it/"+ str(i) + '.txt','a+') 65 f.write(artical) 66 f.close()