1 #!/usr/bin/env python3 2 # -*- coding: utf-8 -*- 3 # author：Momo time:2018/6/30 4 5 """ 6 目标网站：http://tieba.baidu.com/p/3522395718 7 目标内容：跟帖用户名，跟帖内容，跟帖时间 8 涉及知识： 9 Requests 获取网页 10 XPath 提取内容 11 map 实现多线程爬虫 12 掌握以下知识：使用xpath进行网页提取 13 使用map实现多线程爬虫 14 """ 15 16 from lxml import etree 17 from multiprocessing.dummy import Pool as ThreadPool 18 import urllib.request 19 import json 20 # from imp import reload 21 22 # # "将贴吧拷下的代码保存为utf-8" 23 # import sys 24 # reload(sys) 25 # sys.setdefaultencoding('utf-8') 26 27 def towrite(contentdict): 28 f.writelines(u'回帖时间：' + str(contentdict['topic_reply_time']) + '\n' ) 29 f.writelines(u'回帖内容：' + contentdict['topic_reply_content'] + '\n') 30 f.writelines(u'回帖人：' + str(contentdict['user_name']) + '\n\n') 31 32 def spider(url): 33 html_page = urllib.request.urlopen(url) 34 html_code = html_page.read().decode('utf-8') 35 selector = etree.HTML(html_code) 36 contetnt_field = selector.xpath('//div[@class="l_post j_l_post l_post_bright "]') 37 item = {} 38 for each in contetnt_field: 39 reply_info = json.loads(each.xpath('@data-field')[0].replace('"', '')) 40 author = reply_info['author']['user_name'] 41 content = each.xpath('div[@class="d_post_content_main"]/div/cc/div[@class="d_post_content j_d_post_content clearfix"]/text()')[0] 42 reply_time = reply_info['content']['date'] 43 print(content) 44 print(reply_time) 45 print(author) 46 item['user_name'] = author 47 item['topic_reply_content'] = content 48 item['topic_reply_time'] = reply_time 49 towrite(item) 50 51 if __name__ == '__main__': 52 pool = ThreadPool(4) 53 f = open('content.txt', 'a',encoding='utf-8') 54 page = [] 55 for i in range(1,21): 56 newpage = 'http://tieba.baidu.com/p/3522395718?pn=' + str(i) 57 page.append(newpage) 58 59 results = pool.map(spider, page ) 60 pool.close() 61 pool.join() 62 f.close()