批量下载新浪博客文章（改进版）

恩，算是对前段时间写的那个的改进，重写了正则，同时支持翻页下载，还修改了一些bug.但还不支持多线程，打算过几天加上

#!/usr/bin/python
#-*- coding:utf-8 -*-
#****************************

#author:tmyyss
#version:0.2

#****************************

import urllib
import os
import re

def article_format(usock,basedir):    
    title_flag=True
    context_start_flag=True
    context_end_flag=True
    for line in usock:
        if title_flag:
            title=re.findall(r'(?<=>).+(?=<)',line)
            if title:
                title=title[0]
                filename=basedir+title
                print filename
                try:
                    fobj=open(filename,'w+')
                    fobj.write(title+'
')
                    title_flag=False
                except IOError,e:
                    print "Open %s error:%s"%(filename,e)
            else:
                pass
        elif context_start_flag:
            results1=re.findall(r'(<.+?正文开始.+?>)',line)
            if results1:
                context_start_flag=False
        elif context_end_flag:
            results2=re.findall(r'(<.+?正文结束.+?)',line)
            if results2:
                context_end_flag=False
                fobj.write('
END')
                fobj.close()
                break
            else:    
                if 'div' in line or 'span' in line or  '<p>' in line:
                    pass
                else:    
                    line=re.sub('&#65292;',',',line)
                    line=re.sub('&#65306;',':',line)
                    line=re.sub('&#65281;','!',line)
                    line=re.sub('&#65288;','(',line)
                    line=re.sub('&#65289;',')',line)
                    line=re.sub('&#8943;','...',line)
                    line=re.sub('&#65311;','?',line)
                    line=re.sub('&#65307;',';',line)
                    line=re.sub(r'<wbr>','',line)
                    line=re.sub(r'&nbsp;','',line)
                    line=re.sub(r'<brs+?/>','',line)
                    fobj.write(line)
        else:
            print "*****************************************************************"

def parser_page(pageurl):
    total_url=[]
    current_page=get_url(pageurl)
    total_url.extend(current_page)
    usock=urllib.urlopen(pageurl)
        context=usock.read()
    otherpage=re.findall(r'href.+?跳转',context)
        for page in otherpage:
                page=re.findall(r'http.+?html',page)
                pageurl=page[0]
        urllist=get_url(pageurl)
        total_url.extend(urllist)
    return total_url
        
        
def get_url(pageurl):
    urllist=[]
    usock=urllib.urlopen(pageurl)
        context=usock.read()
        raw_url_list=re.findall(r'(<as+title.+?href="http.+?html)',context)
    for url in raw_url_list:
                url=re.findall('(http.+?html)',url)[0]
        urllist.append(url)
    return urllist


if __name__=='__main__':
    basedir='/home/tmyyss/article/'
    if not os.path.exists(basedir):
        os.makedirs(basedir)
    url_list=parser_page("http://blog.sina.com.cn/s/articlelist_1191258123_0_1.html")
    for url in url_list:
        article_usock=urllib.urlopen(url)
        article_format(article_usock,basedir)