一个爬虫从网页中爬取小说

一个简单的实例，从网页中爬取整部小说！

import codecs
import urllib.request
import urllib
import re

urls=[]
url = "https://www.biquger.com/biquge/39691/"
#防爬设置
header = {
   'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36'
}
# 请求
request = urllib.request.Request(url,headers=header)
# 爬取结果
response = urllib.request.urlopen(request)
#decode使用utf-8报错，换成gbk
data = response.read().decode('gbk')

#正则匹配标签
pattern = re.compile("href=['"]([^"'>]*?)['"].*?",re.S)
items = re.findall(pattern, data)
for item in list(items):
    if 'https' in item and '39691' in item:  ##判断是否为http协议链接，并判断是否抓取过
        urls.append(item)
#print (urls)
print (urls[2])
for i in range(2,10):
    request = urllib.request.Request(url=urls[i], headers=header)
     # 爬取结果
    response = urllib.request.urlopen(request)
    #  decode使用utf-8报错，换成gbk
    data = response.read().decode('gbk')
    pattern = re.compile('<!--g0-->.*?<center>', re.S)
    items = re.findall(pattern, data)
    for item in items:
        # 标签过滤
        temp = item.replace('<br />', '')
        temp = temp.replace('&nbsp;', '')
        fo = codecs.open("foo.txt", "a",'utf-8')
        fo.write(temp)
        fo.close()
        print(temp)