requests+xpath+map爬取百度贴吧

# requests+xpath+map爬取百度贴吧
# 目标内容:跟帖用户名,跟帖内容,跟帖时间
# 分解:
# requests获取网页
# xpath提取内容
# map实现多线程爬虫
import requests
from requests.exceptions import RequestException
from lxml import etree
import json
from multiprocessing.dummy import Pool as ThreadPool

def get_html(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        else:
            return None
    except RequestException:
        return None

def parse_html(html):
    selector = etree.HTML(html)
    data = selector.xpath('//div[@class="l_post j_l_post l_post_bright  "]')
    for each in data:
        rs = each.xpath('@data-field')[0]
        rs = json.loads(rs)
        author = rs.get('author').get('user_name')
        author_id = rs.get('content').get('post_id')
        content = each.xpath('div/div/cc/div[@id="post_content_%s"]/text()'% author_id)[0].strip()
        date = rs.get('content').get('date')
        yield {
            'author':author,
            'content':content,
            'date':date
        }

def save_to_txt(result):
    print('正在存储:',result)

    with open('tieba.txt','a',encoding='utf-8') as f:
        f.write('回帖作者:'+result['author']+'
')
        f.write('回帖内容:'+result['content']+'
')
        f.write('回帖时间:'+result['date']+'
')
        f.write('
')


def main(url):
        html = get_html(url)
        if html:
            for result in parse_html(html):
                save_to_txt(result)

if __name__=='__main__':
    
    pool = ThreadPool(4)
    urls=[]
    base_url = 'http://tieba.baidu.com/p/3522395718?pn='
    for page_num in range(1, 21):
        url = base_url + str(page_num)
        urls.append(url)

    pool.map(main,urls)
    pool.close()
    pool.join()