爬取贴吧

在编码问题上困住了好久，对BeautifulSoup的问题在上一篇blog也分析了

最后写文件的时候用的gb18030编码解决的

编码名称	用途
utf8	所有语言
gbk	简体中文
gb2312	简体中文
gb18030	简体中文
big5	繁体中文
big5hkscs	繁体中文

UnicodeEncodeError: 'gbk' codec can't encode character 'xXX' in position XX

哎，我就是载在了GBK手上

from bs4 import BeautifulSoup
from multiprocessing.dummy import Pool as ThreadPool
import requests
import re
import os
# import io
# import sys
import traceback

# sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') #改变标准输出的默认编码

def writeRes(Res):
    filename = 'data/test.txt'
    with open(filename, 'a', encoding = 'gb18030') as f:
        f.write('回帖时间:' + str(Res['date']) + '
')
        f.write('回帖人:' + Res['user_name'] + '
')
        f.write('回帖内容:' + Res['text'] + '

')

def getHTML(url, pages, header):
    try:
        parameters = {'pn':pages}
        r = requests.get(url, params = parameters, headers = header)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        print('网站获取失败')
        return ""

def parse(url):
    #parse every pages
    for pages in range(1, 700):
        try:
            header = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36'}
            html = getHTML(url, pages, header)
            Soup = BeautifulSoup(html, 'html.parser')
            InfoList = Soup.find_all(class_ = re.compile('l_post j_l_post l_post_bright'))
            # turn str into dict
            Res = {}
            for Info in InfoList:
                s = Info.attrs['data-field']
                s = s.replace('null', 'None')
                s = s.replace('true', 'True')
                s = s.replace('false', 'False')
                s = eval(s)
                temp = Info.find(attrs = {'class':'d_post_content'})
                Res['user_name'] = s['author']['user_name']
                Res['date'] = s['content']['date']
                Res['text'] = temp.text.replace(' ', '')
                #print('Hello')
                writeRes(Res)
            print('第{}页解析成功'.format(pages))
        except:
            #traceback.print_exc()
            print('第{}页解析失败'.format(pages))
            continue

def main():
    url = 'http://tieba.baidu.com/p/3522395718'
    parse(url)

if __name__ == '__main__':
    main()