Python 爬虫 之 垃圾爬虫爬了点垃圾小说。。

刚刚开始玩爬虫，想爬点简单的稍微有点意思的，一开始想爬那啥小视频的~但是我还是忍住了哈哈哈。

所以就随便找个静态小说网站，爬了点全本小说。

不多，60几本，平均一本不到十兆的样子。

很慢。。真的很慢。。跑了好久。。改了好几回参数。。忽略可耻的时间。。23号其实是25号。。

中途也出现了很多很多错误。。

当个笑话看看就得了，抛个砖~

毕竟是往前又迈了一步~开心

真的得好好地系统的学习一下爬虫，比如现在我在看 崔庆才，崔先生写的《Python3 网络爬虫开发实战》感觉很棒棒。。要加油！

主要用了 requests，BeautifulSoup，其他还用到了 urljoin 合并网址 

# -*- coding:UTF-8 -*-
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import requests
import sys


class Downloader(object):
    def __init__(self):
        self.server = 'http://www.biquge.com.tw/quanben/'
        self.novel_urls = []
        self.novel_names = []
        self.chapter_urls = []
        self.nums = 0

    def get_novels_info(self):
        req = requests.get(self.server)
        html_doc = req.content.decode('gbk', 'ignore')
        soup = BeautifulSoup(html_doc, 'html.parser')
        spans = soup.find_all('span', class_='s2')
        for span in spans:
            self.novel_urls.append(span.find('a').get('href'))
            self.novel_names.append(span.find('a').get_text())

    def get_download_urls(self, index):
        self.chapter_urls.clear()
        req = requests.get(self.novel_urls[index], timeout=5)
        html_doc = req.content.decode('gbk', 'ignore')
        soup = BeautifulSoup(html_doc, 'html.parser')
        a = soup.find('div', id='list').find_all('a')
        for each in a:
            url = urljoin(self.novel_urls[index], each.get('href'))
            self.chapter_urls.append(url)


    def get_content(self, url):
        req = requests.get(url)
        html_doc = req.content.decode('gbk', 'ignore')
        soup = BeautifulSoup(html_doc, 'html.parser')
        texts = soup.find_all('div', id='content')
        chapter_name = soup.find('div', class_='bookname').find('h1').text
        content = (chapter_name + '

' + texts[0].text).replace('xa0', '')
        return content

    def writer(self, path, content):
        with open(path, 'a', encoding='utf-8') as f:
            f.writelines(content)
            f.write('



')


def main():
    dl = Downloader()
    dl.get_novels_info()
    for i in range(35, len(dl.novel_names)):
        try:
            dl.get_download_urls(i)
            print('开始下载 ' + dl.novel_names[i] + ' ...')
            for j in range(len(dl.chapter_urls)):
                dl.writer(dl.novel_names[i] + '.txt', dl.get_content(dl.chapter_urls[j]))
                sys.stdout.write(" 已下载： %.3f%%" % float(j / len(dl.chapter_urls) * 100) + '
')
                sys.stdout.flush()
            print("
 " + dl.novel_names[i] + " 下载完成。")
        except TimeoutError:
            print(" " + dl.novel_names[i] + "打开失败。")
        except requests.exceptions.ConnectionError:
            print(" " + dl.novel_names[i] + "下载失败。")


if __name__ == '__main__':
    main()