python爬虫--一次爬取小说的尝试

 一次爬取小说的尝试

#!/usr/bin/python
# -*- coding:utf-8 -*-
import requests
from bs4 import BeautifulSoup

url = 'http://www.zanghaihua.org/nanbudangan/'
req = requests.get(url=url)
req.encoding = req.apparent_encoding
soup = BeautifulSoup(req.text,'html.parser')
div = soup.find(name='div',attrs={'class':'booklist'})
# print(div)
span_list = div.find_all('span')
# print(span_list)

for span in span_list:
    a = span.find('a')
    # span_text=span.find(attrs={'class':'v'})
    if not a:
        continue
    a_url = a.get('href')
    # a_text = a.text


    response = requests.get(url=a_url)
    response.encoding =response.apparent_encoding
    # print(response.text)
    # print(response.encoding)
    soup = BeautifulSoup(response.text,'html.parser')

    Bookname = soup.find(name='h1',attrs={'align':'center'}).text
    # print('书名：%s' %Bookname)
    ChapterTitle =soup.find(name='div',attrs={'class':'chaptertitle'}).text
    # print('章节名：%s' %ChapterTitle)

    Title = soup.find(name='div',attrs={'id':'BookText'}).get_text('
','<br/><br/>')
    #用get_text获取文本并将<br/><br/>替换成

    # print(Title)

    with open(Bookname,'ab+') as f:
        #以追加模式写入文件

        if ChapterTitle=='关于南部档案馆的研究':
            f.write(Bookname.encode('utf-8'))
        f.write(ChapterTitle.encode('utf-8'))
        f.write(Title.encode('utf-8'))