爬取酷狗榜单并可视化词云 Python

爬取酷狗音乐榜单并做一个词云展示。分为两部分，爬虫部分和可视化部分；爬虫的话我用的是openpyxl进行保存的，因为这块一直是薄弱的地方，加强一下面向对象保存数据到excel里。

import openpyxl
import requests
import parsel


def create_workbook():
    wb = openpyxl.Workbook() # 创建工作簿
    ws = wb.create_sheet(title='summary', index=0)
    wb.remove(wb['Sheet']) #  删除原始表
    wb.close()
    wb.save('202201122test.xlsx')


# 打开工作簿并写入头
def open_workbook_and_write_header():
    wb = openpyxl.load_workbook('202201122test.xlsx')
    ws = wb['summary']  # 选中工作表
    ws.append(['歌名', '歌手', '链接', '时长'])
    wb.close()
    wb.save('anothertest.xlsx')


class kugouSpider():
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36',
    }
    def __init__(self, url, headers=headers):
        self.url = url
        self.headers = headers

    def get_html(self):
        response = requests.get(url=self.url, headers=self.headers)
        return response.text

    def parsel_data(self):
        # wb = openpyxl.load_workbook('anothertest.xlsx')
        # ws = wb['summary']
        songNames = []
        singers = []
        playPages = []
        ttimes = []
        selector = parsel.Selector(self.get_html())
        results = selector.xpath('//div[@id="rankWrap"]/div[2]/ul/li')
        for item in results:
            songName = item.xpath('.//a/@title').get().split('-')[0]
            songNames.append(songName)
            singer = item.xpath('.//a/@title').get().split('-')[1]
            singers.append(singer)
            playPage = item.xpath('.//a/@href').get()
            playPages.append(playPage)
            ttime = item.xpath('.//span[@class="pc_temp_time"]/text()').get().strip()
            ttimes.append(ttime)
            # ws.append([songName, singer, playPage, ttime])
            # print(songName, singer, playPage, ttime, sep=' | ')
        return zip(songNames, singers, playPages, ttimes)
        # wb.close()
        # wb.save('anothertest.xlsx')

    def save_to_excel(self):
        # 读取excel并保存数据
        wb = openpyxl.load_workbook('anothertest.xlsx')
        ws = wb['summary']
        for songName, singer, playPage, ttime in self.parsel_data():
            print(songName, singer, playPage, ttime, sep=' | ')
            ws.append([songName, singer, playPage, ttime])
        wb.close()
        wb.save('anothertest.xlsx') # 每爬取一页就保存一次

    def run(self):
        self.save_to_excel()

if __name__ == "__main__":
    # 方便演示写了两个函数，这样步骤也清晰
    create_workbook()  # 创建工作簿
    open_workbook_and_write_header()  # 写入头
    for page in range(1, 20+1):
        url = f'https://www.kugou.com/yy/rank/home/{page}-8888.html?from=rank'
        app = kugouSpider(url=url)
        app.run()

保存到excel不像保存到csv，csv相对简单很多，个人感觉。excel有打开保存的过程，一不小心就没保存到数据，或者翻页爬取的时候，保存的是最后一页的数据，不像csv，csv的话，只要打开文档用a+模式就可以完美解决翻页的问题。

如果偏执xlsx格式文件而又想简单的写入数据保存到本地，可以先将数据保存成csv，然后用pandas将其转换成excel文件。

数据转换部分：从excel读取数据有xlrd和pandas，两种方法都贴一下，很简单。

def singer_process_by_xlrd():
    """
    通过xlrd读取excel文档并取出歌手名返回列表
    """
    singers = []
    xlsx = xlrd.open_workbook('anothertest.xlsx')
    table = xlsx.sheet_by_name('summary')
    # 遍历表格行
    for row in range(1, table.nrows): # 从表头的下一行开始遍历
        singer = table.cell(row, 0).value # 读取第row行，第一列的值
        singers.append(singer)
    return singers

上面的是xlrd的读取excel表格。下面用pandas。

def singer_process_by_pandas():
    """
    通过pandas读取excel文档并取出歌手名返回列表
    """
    df = pd.read_excel('anothertest.xlsx') # 读取文档
    singers = df['歌名'].tolist()
    return singers

数据处理完了，现在来进行绘制词云：

第一是根据stylecloud的制作，stylecloud需要用jieba进行分词然后再进行绘制：

def gen_style_words():
    wordsList = singer_process_by_pandas() # 列表
    wordsStr = ' '.join(wordsList) # 转换成字符串
    # 开始分词
    wordResults = jieba.cut(wordsStr)
    print(wordResults) # 是个生成器
    newResults = ''.join(wordResults) #
    print(newResults)
    newResultsStr = re.sub(r"[A-Z0-9-a-zăâ、()\!\%\[\]\,\。]", "", newResults) # 将特殊字符替
    print(newResultsStr)

    with open('stopwords.txt', mode='r', encoding='utf-8') as f:
        stopWords = f.read() # str
        # print(type(stopWords))
        stopWords = stopWords.split('\n') #list
        # print(type(stopWords))
    stylecloud.gen_stylecloud(
        text=newResultsStr,
        size=1280,
        font_path=r'C:\Windows\Fonts\simhei.ttf',
        max_font_size=200,
        max_words=150,
        # custom_stopwords=stopWords,
        output_name = 'str.png',
    )

第二种是wordcloud，wordcloud分词的时候，只需要一个str，所以将列表转换成str直接进行绘制。

def word_cloud_style():
    """
    用wordcloud其实没有用到结巴分词，它只需要一个str，如果要精准分词，可以用jieba分一次再用wordcloud生成
    """
    singers = singer_process_by_xlrd() # 列表
    singerStr = re.sub(r"[A-Z0-9-a-zăâ、()\!\%\[\]\,\。]", "", ''.join(singers))
    print(singerStr)
    # # 开始分词
    # words = jieba.cut(singerStr)
    # print(words) # 生成器
    # wordStr = ''.join(words)
    img = Image.open('wc.jpg')
    img_array = np.array(img)
    wordcloud = WordCloud(
        background_color='white',
        font_path=r'C:\Windows\Fonts\simhei.ttf',
        width=1280,
        height=960,
        max_words=150,
        max_font_size=200,
        scale=10,
        margin=2,
        mask=img_array,
        collocations=False,
    ).generate(singerStr)
    plt.imshow(wordcloud)
    plt.axis('Off')
    plt.show()
    wordcloud.to_file('wordcloud.jpg')

调用函数就可以绘制。

效果图：