python爬取小说

运行结果：

代码：

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
import os
 
class NovelSpider:
    def __init__(self):
        self.start_url = 'https://www.biqukan.com/1_1680/'
 
    def get_novel(self):
        response = requests.get(self.start_url)
        soup = BeautifulSoup(response.text, 'html.parser')
        div_chapter = soup.find(class_="listmain")
        chapter_list = div_chapter.find_all('a')
        chapter_list = chapter_list[12:]
        chapter = []
        chapter_num = len(chapter_list)
        count = 0
        print('《凡人修仙传仙界篇》开始下载:')
        for cl in chapter_list:
            chapter_dict = {}
            chapter_name = cl.get_text()
            chapter_dict['name'] = chapter_name
            chapter_url = cl.get('href')
            chapter_dict['value'] = 'https://www.biqukan.com' + chapter_url
            if chapter_dict not in chapter:
                chapter.append(chapter_dict)
            print(f"已下载:{count}/{chapter_num}")
            self.download_novel(chapter_dict)
            count += 1
 
    def parse_novel(self, url):
        browser = webdriver.PhantomJS(executable_path=r'F:Spider
ovelSpiderphantomjs.exe')
        browser.get(url)
        soup = BeautifulSoup(browser.page_source, 'html.parser')
        find_txt = soup.find(class_='showtxt')
        # print(type(find_txt.get_text()))
        return find_txt.get_text()
 
    def download_novel(self, data): 
        filename = data['name']
        url = data['value']
        txt = self.parse_novel(url)
 
        path = r"F:Spider
ovelSpider"
        isExists = os.path.exists(path)
        if not isExists:
            os.mkdir(path)
        else:
            pass
 
        with open(path + f'凡人修仙传仙界篇.txt', 'a', encoding='utf-8') as f:
            f.write(f'{filename}

')
            f.write(txt)
            f.write('
======

')
            f.close()
 
if __name__ == '__main__':
    ns = NovelSpider()
    ns.get_novel()