Python爬虫(三)--百度贴吧

#coding=utf-8
import urllib2
import urllib
import re
# 处理页面标签类
class Tool:
    # 去除img标签，7位长空格
    removeImg = re.compile(r'<img.*?>| {7}|')
    # 删除超链接标签
    removeAddr = re.compile('<a.*?>|</a>')
    # 把换行标签换位
    replaceLine = re.compile('<tr>|<div>|</div|</p>')
    # 将制表<td>换位	
    replaceTD = re.compile('<td>')
    # 将段落开头换为
加两空格
    replacePara = re.compile('<p.*?>')
    # 将换行符或双换行符换为
    replaceBR = re.compile('<br><br>|<br>')
    # 删除其他标签
    removeExtraTag = re.compile('<.*?>')
    def replace(self,x):
        x = re.sub(self.removeImg,"",x)
        x = re.sub(self.removeAddr,"",x)
        x = re.sub(self.replaceLine,"
",x)
        x = re.sub(self.replaceTD,"	",x)
        x = re.sub(self.replacePara,"
",x)
        x = re.sub(self.replaceBR,"
",x)
        x = re.sub(self.removeExtraTag,"",x)
        # 将前后多余的内容删除
        return x.strip()
# 百度贴吧爬虫类
class BDTB:
    # 初始化，传入基地址，传入是否“只看楼主”参数
    def __init__(self,baseUrl,seeLz,floorTag):
        # base连接地址
        self.basURL = baseUrl
        # 是否只看楼主
        self.seeLZ = '?see_lz='+str(seeLz)
        # 工具类
        self.tool = Tool()
        # 全局file变量，文件写入操作对象
        self.file = None
        # 楼层标号,初始为1
        self.floor = 1
        # 默认的标题
        self.defaultTitle = u"百度贴吧"
        # 是否写入楼分隔符的标记
        self.floorTag = floorTag
    # 获取该页(pageNum)的代码
    def getPage(self,pageNum):
        try:
            url = self.basURL + self.seeLZ + '&pn=' + str(pageNum)
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            # 返回utf-8格式编码内容
            return response.read().decode('utf-8')
        except urllib2.URLError,e:
            if hasattr(e,"reason"):
                print u"连接百度贴吧失败，错误原因",e.reason
                return None
    # 获得页面标题
    def getTitle(self,page):
        # 获取标题的正则表达式
        pattern = re.compile(r'<h3 class="core_title_txt.*?>(.*?)</h3>',re.S)
        result = re.search(pattern, page)
        if result:
            return result.group(1).strip()
        else:
            return None
    # 提取页数
    def getPageNum(self,page):
        # 获取页数的正则表达式
        pattern = re.compile(r'reply_num.*?</span.*?class="red">(.*?)</span>',re.S)
        result = re.search(pattern,page)
        if result:
            return result.group(1).strip()
        else:
            return None
    # 提取内容
    def getContent(self,page):
        pattern = re.compile(r'post_content.*?>(.*?)</div>',re.S)
        items = re.findall(pattern,page)
        contents = []
        for item in items:
            #将文本内容进行处理，同时在前后加上换行符
            content = "
" + self.tool.replace(item)+"
"
            contents.append(content.encode('utf-8'))
        return contents
    def setFileTitle(self,title):
        if title is not None:
            self.file = open(title+".txt","w+")
        else:
            self.file = open(self.defaultTitle+".txt","w+")
    # 写文件
    def writeData(self,contents):
        for item in contents:
            if self.floorTag == "1":
                floorLine = "
"+str(self.floor)+u"--------------------------------------
"
                self.file.write(floorLine)
            self.file.write(item)
            self.floor += 1
    def start(self):
        indexPage = self.getPage(1)
        pageNum = self.getPageNum(indexPage)
        title = self.getTitle(indexPage)
        self.setFileTitle(title)
        if pageNum == None:
            print  u"URL已失效，请重试"
            return
        try:
            print "该帖子共有"+str(pageNum)+"页。"
            for i in range(1,int(pageNum)+1):
                print "正在写入第"+str(i)+"页数据"
                page = self.getPage(i)
                contents = self.getContent(page)
                self.writeData(contents)
        except IOError,e:
            print "写入异常，原因:"+e.message
        finally:
            print "写入完成"
baseURL = 'http://tieba.baidu.com/p/3138733512'
seeLz = raw_input("是否只获取楼主发言，是输入1，否输入0
")
floorTag = raw_input('是否写入楼层信息，是输入1，否输入0
')
bdtb = BDTB(baseURL,seeLz,floorTag)
bdtb.start()

来自为知笔记(Wiz)