from urllib import request
from bs4 import BeautifulSoup
import webbrowser
if __name__ == "__main__":
url= 'http://www.biqukan.com/47_47404/17679470.html'
webbrowser.open(url)
head = {}
head['User-Agent'] = 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19'
requ = request.Request(url, headers = head)
download_response = request.urlopen(requ)
download_html = download_response.read().decode('gbk','ignore')
soup_texts = BeautifulSoup(download_html, 'lxml')
texts = soup_texts.find_all(id = 'content', class_ = 'showtxt')
soup_text = BeautifulSoup(str(texts), 'lxml')
#将xa0无法解码的字符删除
download_text = soup_text.div.text.replace('xa0','')
#将爬取结果写入文档中
with open("爬虫.txt","w") as filename:
filename.write(download_text)