python应用：爬虫实例(静态网页)

爬取起点中文网某本小说实例：

# -*-coding:utf8-*-
import requests
import urllib
import urllib2
from bs4 import BeautifulSoup
import sys
reload(sys)
sys.setdefaultencoding('utf-8')       #读写中文内容

class QDZW:
    def __init__(self, url):
        self.url = url           #网址
        self.user_agent = 'Mozilla/5.0 (Windows NT 6.1; (Windows NT 6.1; WOW64; rv:62.0) Gecko/20100101 Firefox/61.0'      #请求头属性（模拟浏览器访问）
        self.headers = {'User-Agent' : self.user_agent}

    #获取网址静态HTML（如果未访问成功，则持续访问，直到访问成功为止）
    def gethtml(self):
        
            try:
                #return requests.get(self.url，headers = self.headers,time_out = 2)
                request = urllib2.Request(self.url,headers = self.headers)
                response = urllib2.urlopen(request,time_out = 2)    #设置访问时长限制
                temp = False
                return response
            except Exception as e:
                print e
    #解析HTML内容，获取需要的内容           
    def novel(self):
        while True:
            r = self.gethtml()
            while r == 'None':         #r内容为空表示访问超时，则继续访问
                r = self.gethtml()     
            #print r.encoding    #输出编码类型
            soup = BeautifulSoup(r ,"html.parser")     #利用BeautifulSoup解析网址内容，class前加 . id前加 #
            #select输出内容为数组，select参数从body中一级结点开始即可，可逐级叠加定位
            #XXX.text为读取标签中的内容  XXX.attrs['xx']为读取 XXX中属性为xx的值 ，XXX为HTML格式的文本
            section_name = soup.select(' .wrap #j_readMainWrap #j_chapterBox .text-wrap .main-text-wrap .text-head .j_chapterName')[0].text  
            print section_name
            section_text = soup.select('.wrap #j_readMainWrap #j_chapterBox .text-wrap .main-text-wrap .read-content')[0].text
            section_text = self.spliting(section_text)
            #print section_text
            self.writing(section_name + "
" + section_text)

            next_url = soup.select('.wrap #j_readMainWrap .chapter-control #j_chapterNext')[0]
            #print next_url.text 
            if next_url.text == "下一章":
                self.url = "https:" + next_url.attrs['href']
            else:
                break
    #在去掉<p></p>的位置添加换行符号“
”
    def spliting(self, string):
        str_split = string.split()  #以split()中参数为标准，切割字符串
        string = ""
        for text in str_split:
            if text != "":
                string = string + "    " + text + "
"
        return string
    #输出读取的novel内容
    def writing(self, novel):
        outputs = open(unicode('至尊剑皇.txt','utf8'),'a')      #a为追加型写入
        outputs.write(novel)
        outputs.close()
        
qdzw = QDZW('https://read.qidian.com/chapter/JDViC81SWM41/pskSqRrpbXDgn4SMoDUcDQ2')     #参数为初始网址  
qdzw.novel()