17 中国天气网信息爬取

"""中国天气网爬虫"""

import requests
from bs4 import BeautifulSoup

HEADERS = {
    'User-Agent': 'Mozilla/5.0',
}

def parse_detail_page(url, is_html5lib):
    """爬取具体页面具体数据"""

    respose = requests.get(url, headers=HEADERS)
    text = respose.content.decode('utf-8')
    # with open('weather.html', 'w', encoding='utf-8') as fp:
    #     fp.write(text)
    if is_html5lib == False:
        soup = BeautifulSoup(text, 'lxml')
    else:
        soup = BeautifulSoup(text, 'html5lib')
    # 以下为具体爬取数据方法
    conMidtab = soup.find_all('div', attrs={'class':'conMidtab'})
    tables = conMidtab[0].find_all('table')
    for table in tables:
        trs = table.find_all('tr')[2:]
        for index,tr in enumerate(trs):
            tds = tr.find_all('td')
            city_td = tds[0]
            if index == 0:
                city_td = tds[1]
            city = list(city_td.stripped_strings)[0]
            temp_td = tds[-2]
            min_temp = list(temp_td.stripped_strings)[0]
            # 输出城市及其最低温度
            print({'city': city, 'min_temp': min_temp})

    print("="*40)

def get_detail_urls(url, base_url):
    """得到华北、东北、华东、华中、华南、西北、西南、港澳台的具体页面链接"""

    urllists = []       # 具体的页面信息列表
    respose = requests.get(url, headers=HEADERS)
    text = respose.content.decode('utf-8')
    soup = BeautifulSoup(text, 'lxml')
    # 数据爬取
    uls = soup.find_all('ul', class_='lq_contentboxTab2')
    alists = uls[0].find_all('a')
    for list in alists:
        newurl = base_url + list['href']
        urllists.append(newurl)

    return urllists

def spider():
    """"""

    # 初始爬取页面
    src_url = "http://www.weather.com.cn/textFC/hb.shtml"
    base_url = "http://www.weather.com.cn"
    urllists = []
    urllists = get_detail_urls(src_url, base_url)
    #print(urllists)
    is_html5lib = False     # 爬取页面是否用html5lib库
    for index,urllist in enumerate(urllists):
        if index != len(urllists)-1:
            parse_detail_page(urllist, is_html5lib)
        else:
            is_html5lib = True
            # url = "http://www.weather.com.cn/textFC/gat.shtml"这个页面需要用html5lib库解析，不然数据有错
            parse_detail_page(urllist, is_html5lib)

if __name__ == '__main__':
    spider()