爬大主宰小说（第一代）

功能有待加强，可实现抓取，不过速度太慢，由于代理，没设置header，幸好网站对爬虫没设限制

import requests
from bs4 import BeautifulSoup
def get_url_list(url):
    content = requests.get(url).content
    soup = BeautifulSoup(content,'lxml')
    list = []
    for i in soup.select('#list dl dd a'):
                         temp = 'http://www.biquge.info/0_921/'+i.get('href')
                         list.append(temp)
    return list
def get_date(url):
    content = requests.get(url).content
    soup = BeautifulSoup(content,'lxml')
    soup1 = str(soup.select('#content'))
    text = soup1.replace('<br/>','
').replace('</div>','
').replace('<div id="content">','')
    title = soup.select('.content_read .box_con .bookname h1')[0].get_text()
    f = open(r'F:\栋歌第一代爬虫.txt','a+',encoding = 'utf-8')
    f.write(title +"

"+text)
    print(title)
    f.close()
                         
if __name__=="__main__":
    url = 'http://www.biquge.info/0_921/'
    url_list = get_url_list(url)
    for i in url_list:
        get_date(i)