爬虫批量自动下载小说

下载排行榜的所有小说

#!/usr/bin/env python
# -*- coding:utf-8 -*- 
#Author: ss

from bs4 import BeautifulSoup
import requests
import time
import os

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36'
}

def get_text(url,title1):
    #url = 'https://www.xxbiquge.com/0_36/8840634.html'
    data = requests.get(url,headers=headers)
    time.sleep(0.5)
    soup = BeautifulSoup(data.text.encode('ISO-8859-1').decode('utf-8'),'lxml')
    text = soup.select('div.content_read > div > div#content')[0].text
    title2 = soup.select('div.content_read > div > div.bookname > h1')[0].text
    ls = []
    for i in text:
        if i in "' 
','xa0','readx();'":
            continue
        else:
            ls.append(i)
    text = ''.join(ls)
    with open('.\books\' + title1 + '.txt','ab+') as f:
        f.write((title1 + '
').encode())
        #f.write('
'.encode())
        f.write(text.encode())
        f.write('

'.encode())
    print('正在下载{}'.format(title2))

def get_one_links(url):
    #url = 'https://www.xxbiquge.com/0_36/'
    data = requests.get(url, headers=headers)
    soup = BeautifulSoup(data.text.encode('ISO-8859-1').decode('utf-8'), 'lxml')
    links = soup.select('div#list > dl > dd')
    title = soup.select('div#maininfo > div#info > h1')[0].text
    print('开始下载{}'.format(title))
    for i in links:
        data = i.select('a')
        for m in data:
            url = 'https://www.xxbiquge.com' + m.get('href')
            get_text(url,title)

def get_all():
    url = 'https://www.xxbiquge.com/xbqgph.html'
    data = requests.get(url,headers=headers)
    time.sleep(0.5)
    soup = BeautifulSoup(data.text.encode('ISO-8859-1').decode('utf-8'),'lxml')
    links = soup.select('div.novelslist2 > ul > li')
    for i in links:
        data = i.select('span.s2 > a')
        for m in data:
            url = 'https://www.xxbiquge.com' + data[0].get('href')
            get_one_links(url)

if not os.path.exists('.\books'):
    os.mkdir('.\books')
get_all()