爬取微博热搜榜 李白之死 Python

最近有关中国传统文化的内容频频登上热搜，就比如最近的李白之死，今天换一种方式爬取，以前爬取微博评论是网址里一大串参数，今天把参数提出来做一个字典，然后请求的时候再构造url。

"""
    就爬取李白之死的评论
"""
import requests
import re
import openpyxl as op

wb = op.Workbook()
ws = wb.create_sheet(index=0)
# 表头
ws.cell(row=1, column=1, value='评论者id') # 第一行第一列userId
ws.cell(row=1, column=2, value='评论者昵称') # 第一行第一列userId
ws.cell(row=1, column=3, value='获赞数') # 第一行第一列userId
ws.cell(row=1, column=4, value='创建时间') # 第一行第一列userId
ws.cell(row=1, column=5, value='评论内容') # 第一行第一列userId

headers = {
    "cookie": "cookie",
    "referer": "https://m.weibo.cn/status/L690FmKXW?jumpfrom=weibocom",
    "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
}
page = 1
while page < 100 + 1:
    url = 'https://m.weibo.cn/comments/hotflow'
    if page == 1:
        params = {
                    'id': '4714646055423756',
                    'mid': '4714646055423756',
                    'max_id_type':0,
        }
    else:
        params = {
                    'id': '4714646055423756',
                    'mid': '4714646055423756',
                    'max_id': max_id,
                    'max_id_type':max_id_type,
        }
    response = requests.get(url=url, headers=headers, params=params)
    max_id = response.json()['data']['max_id']
    max_id_type = response.json()['data']['max_id_type']
    results = response.json()['data']['data'] # 获取到评论列表
    for item in results:
        userId = item['user']['id']
        userName = item['user']['screen_name']
        likeCount = item['like_count']
        timeCreated = item['created_at'] # 创建时间
        commentContent = item['text'] # 评论内容
        print(userId, userName, likeCount, timeCreated, commentContent, response.url, sep=' | ')
        ws.append([userId, userName, likeCount, timeCreated, commentContent])
    page += 1

wb.save('李白之死.xlsx')
wb.close()

保存方式有很多，前面也有过一篇关于python爬虫数据保存方式的。但是今天只要爬评论内容来做个词频。

"""
    就爬取李白之死的评论
"""
import requests
import re
import openpyxl as op

# wb = op.Workbook()
# ws = wb.create_sheet(index=0)
# 表头
# ws.cell(row=1, column=1, value='评论者id') # 第一行第一列userId
# ws.cell(row=1, column=2, value='评论者昵称') # 第一行第一列userId
# ws.cell(row=1, column=3, value='获赞数') # 第一行第一列userId
# ws.cell(row=1, column=4, value='创建时间') # 第一行第一列userId
# ws.cell(row=1, column=5, value='评论内容') # 第一行第一列userId

headers = {
    "cookie": "cookie",
    "referer": "https://m.weibo.cn/status/L690FmKXW?jumpfrom=weibocom",
    "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
}
page = 1
while page < 100 + 1:
    url = 'https://m.weibo.cn/comments/hotflow'
    if page == 1:
        params = {
                    'id': '4714646055423756',
                    'mid': '4714646055423756',
                    'max_id_type':0,
        }
    else:
        params = {
                    'id': '4714646055423756',
                    'mid': '4714646055423756',
                    'max_id': max_id,
                    'max_id_type':max_id_type,
        }
    response = requests.get(url=url, headers=headers, params=params)
    max_id = response.json()['data']['max_id']
    max_id_type = response.json()['data']['max_id_type']
    results = response.json()['data']['data'] # 获取到评论列表
    for item in results:
        """
        这一次只爬评论内容
        """
        commentContent = re.sub(r'<[^>]*>', '', item['text']) # 将评论内容里的特殊字符用正则替换掉
        print(commentContent)
        with open('libazhisi.txt', mode='a', encoding='utf-8') as f:
            f.write(f'{commentContent}\n') # 换行写入
    page += 1

# wb.save('李白之死.xlsx')
# wb.close()

词频展示：

"""
    做个词频
"""
# 打开文档
import re
from collections import Counter
import jieba
from pyecharts.charts import Bar
import pyecharts.options as opts
from pyecharts.globals import ThemeType

def replaceSth(sth):
    pattern = re.compile(r'[a-zA-Z0-9…，\@”！[\\]_]。')
    new = re.sub(pattern, '', sth)
    return new

with open('李白之死.txt', mode='r', encoding='utf-8') as f:
    reader = f.read()
    new_reader = re.sub('[”0-9a-zA-Z！？，\@\-\_/\[\]\\...\\#【】\*\…。\“]', '', reader)
    # print(new_reader)
# 打开停用此表
with open('stopwords.txt', mode='r', encoding='utf-8') as f:
    stopReader = f.read()
word = jieba.cut(new_reader) # 分词
words = [] # 空list
for item in list(word): # 遍历
    if item not in stopReader: # 条件判断
        words.append(item) # 追加列表

# print(dict(Counter(words))) # 打印分词
# 开始画图
label = [] # x轴标签
data = [] # y轴数据
for k, v in dict(Counter(words).most_common(10)).items(): # 取字典里常出现的十个
    label.append(k)
    data.append(v)

bar = (
    Bar(init_opts=opts.InitOpts(width='1080px', height='960px', theme=ThemeType.LIGHT))
    .add_xaxis(label)
    .add_yaxis(series_name='', y_axis=data)
    .set_global_opts(
        title_opts=opts.TitleOpts(
            title='李白之死词频',
            title_textstyle_opts=opts.TextStyleOpts(font_size=30),),
        toolbox_opts=opts.ToolboxOpts(),
        tooltip_opts=opts.TooltipOpts(
            is_show=True,
            trigger='axis',
            axis_pointer_type='cross',
        ),
    )
).render('词频.html')