Python爬虫之利用BeautifulSoup爬取豆瓣小说（三）——将小说信息写入文件

#-*-coding:utf-8-*-
import urllib2
from bs4 import BeautifulSoup

class dbxs:

    def __init__(self):
        self.pageIndex = 0
        self.enable = True
        self.file = None
        self.content = []
        
        
    #获取html页面的内容
    def getPage(self, pageIndex):
        try:
            #设置代理ip
            enable_proxy = True
            proxy_handler = urllib2.ProxyHandler({'Http': '113.118.170.230:808'})
            null_proxy_handler = urllib2.ProxyHandler({})
            if enable_proxy:
                opener = urllib2.build_opener(proxy_handler)
            else:
                opener = urllib2.build_opener(null_proxy_handler)
            urllib2.install_opener(opener)
            #获得页面响应的内容
            url = 'https://www.douban.com/tag/%E5%B0%8F%E8%AF%B4/book' + "?start=" + str(pageIndex)
            #设置请求头部信息，模拟浏览器的行为
            my_headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:55.0)'}
            request =   urllib2.Request(url, headers = my_headers)
            response = urllib2.urlopen(request)
            return response.read()
        except urllib2.URLError, e:
            if hasattr(e, "code"):
                print e.code
            if hasattr(e, "reason"):
                print e.reason
                return None

    #过滤查找这一页的小说名字，信息和评分
    def getContent(self, pageIndex, content):
        pageCode = self.getPage(pageIndex)
        soup = BeautifulSoup(pageCode, 'html.parser')
        #在获得相应的内容中找出所有标签为<dd>的内容（里面包含了我们需要的小说信息）
        contents = soup.find_all('dd')
        
        if contents:
            for item in contents:
                title = item.find(class_ = 'title').string.encode('utf-8')
                info = item.find(class_ = 'desc').string.strip().encode('utf-8')
                rate = item.find(class_ = 'rating_nums')
                #通过试验，我们发现某一页可能存在小说没有评分，如果我们不判断rate，那么可能就出现报错
                if rate:
                    rates = rate.string.encode('utf-8')
                    content.append([title, info, rates])
                    
                else:
                    content.append([title, info])
        #如果页面不包含<dd>标签，我们应该停止
        else:
            print u"所有页面已加载完"
            self.enable = False

        return content
        
            

    #写入文件
    def writeData(self, content):
        self.file = open("bdxs.txt", "w+")   #必须在for循环外面，不然每一次写入都会覆盖之前的数据
        for item in content:
            if len(item) == 3:
                self.file.write(item[0] + "
")
                self.file.write(item[1] + "
")
                self.file.write(u"评分:" + item[2] + "

")
            else:
                self.file.write(item[0] + "
")
                self.file.write(item[1] + "
")
            self.file.write("========================================

")

            
    #创建一个开始方法
    def start(self):
        x = 1
        while self.enable == True:
            content = self.getContent(self.pageIndex, self.content)
            if self.enable == True:
                print "正在写入第%s页..." %x
            self.writeData(content)
            self.pageIndex += 15
            x += 1
                      
            
DBXS = dbxs()
DBXS.start()

这段代码我还没理解透彻，比如每一页的小说信息写入完成后，怎么在后面加上第几页，后期我将继续完善它。