18 中国天气网信息爬取（排序并可视化显示）

"""中国天气网爬虫"""

import requests
from bs4 import BeautifulSoup
from pyecharts import Bar


HEADERS = {
    'User-Agent': 'Mozilla/5.0'
}

ALL_DATA = []       # 所有爬取的数据

def parse_detail_page(url, is_html5lib):
    """爬取具体页面具体数据"""

    respose = requests.get(url, headers=HEADERS)
    text = respose.content.decode('utf-8')
    # with open('weather.html', 'w', encoding='utf-8') as fp:
    #     fp.write(text)
    if is_html5lib == False:
        soup = BeautifulSoup(text, 'lxml')
    else:
        soup = BeautifulSoup(text, 'html5lib')
    # 以下为具体爬取数据方法
    conMidtab = soup.find_all('div', attrs={'class':'conMidtab'})
    tables = conMidtab[0].find_all('table')
    for table in tables:
        trs = table.find_all('tr')[2:]
        for index,tr in enumerate(trs):
            tds = tr.find_all('td')
            city_td = tds[0]
            if index == 0:
                city_td = tds[1]
            city = list(city_td.stripped_strings)[0]
            temp_td = tds[-2]
            min_temp = list(temp_td.stripped_strings)[0]
            # 存储爬取数据，把最低温度转成整型，排序需要
            ALL_DATA.append({'city': city, 'min_temp': int(min_temp)})

def data_visualization():
    """取出前10最低温度并用柱状图显示"""

    # ALL_DATA.sort(key=lambda data:data['min_temp'], reverse=True)
    ALL_DATA.sort(key=lambda data: data['min_temp'])
    #print(ALL_DATA)
    #print(len(ALL_DATA))
    # 获取前10
    data = ALL_DATA[0:10]
    print(data)
    cities = list(map(lambda x:x['city'], data))
    temps = list(map(lambda x:x['min_temp'], data))
    chart = Bar("中国天气最低气温排行榜")
    chart.add('', cities, temps)
    chart.render('weather.html')


def get_detail_urls(url, base_url):
    """得到华北、东北、华东、华中、华南、西北、西南、港澳台的具体页面链接"""

    urllists = []       # 具体的页面信息列表
    respose = requests.get(url, headers=HEADERS)
    text = respose.content.decode('utf-8')
    soup = BeautifulSoup(text, 'lxml')
    # 数据爬取
    uls = soup.find_all('ul', class_='lq_contentboxTab2')
    alists = uls[0].find_all('a')
    for list in alists:
        newurl = base_url + list['href']
        urllists.append(newurl)

    return urllists

def spider():
    """"""

    # 初始爬取页面
    src_url = "http://www.weather.com.cn/textFC/hb.shtml"
    base_url = "http://www.weather.com.cn"
    urllists = []
    urllists = get_detail_urls(src_url, base_url)
    #print(urllists)
    is_html5lib = False     # 爬取页面是否用html5lib库
    for index,urllist in enumerate(urllists):
        if index != len(urllists)-1:
            parse_detail_page(urllist, is_html5lib)
        else:
            is_html5lib = True
            # url = "http://www.weather.com.cn/textFC/gat.shtml"这个页面需要用html5lib库解析，不然数据有错
            parse_detail_page(urllist, is_html5lib)

    # 排序并可视化数据
    data_visualization()

if __name__ == '__main__':
    spider()

注意：需要安装 pyecharts

建议 pip intstall pyecharts==0.1.9.5

https://blog.csdn.net/Nurbiya_K/article/details/105354670

https://www.jianshu.com/p/b718c307a61c