爬虫_中国天气网_文字天气预报（xpath）

import requests
from lxml import etree

headers = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'

def get_html(url):
    try:
        html = requests.get(url, headers={'User-Agent':'headers'})
        html.encoding = html.apparent_encoding
        if html.status_code == 200:
            return html.text
        return 0
    
    except RequestsException:
        return 0
'''
conMintab:  华北  
    conMintab2    北京
    conMintab2    天津
    conMintab2    河北
    conMintab2    山西
    conMintab2    内蒙古


//div[@class="conMidtab"][1] 今天
//div[@class="conMidtab"][2] 明天 
                ...
//div[@class="conMidtab"][7] 最后一天
'''

def parse_html(html):
    wea = []
    html_element = etree.HTML(html)
    # !!!!!  trs = html_element.xpath('//div[@class="conMidtab"][1]//tr')[2:]
    provinces = html_element.xpath('//div[@class="conMidtab"][1]//div[@class="conMidtab2"]')
    for province in provinces:
        trs = province.xpath('.//tr')[2:]
        for tr in trs:
            weather = {}
            city = tr.xpath('.//td[@width="83"]/a/text()')
            phenomenon = tr.xpath('.//td[@width="89"]/text()')
            wind = tr.xpath('.//td[@width="162"]//text()')
            hightest = tr.xpath('.//td[@width="92"]/text()')
            weather['city'] = city
            weather['phenomenon'] = phenomenon
            weather['wind'] = wind
            weather['hightest'] = hightest
            while '
' in wind:
                wind.remove('
')
            wea.append(weather)
        
    print(wea)    
    
        
def main():
    urls = ['http://www.weather.com.cn/textFC/hb.shtml',
            'http://www.weather.com.cn/textFC/db.shtml',
            'http://www.weather.com.cn/textFC/hd.shtml',
            'http://www.weather.com.cn/textFC/hz.shtml',
            'http://www.weather.com.cn/textFC/hn.shtml',
            'http://www.weather.com.cn/textFC/xb.shtml',
            'http://www.weather.com.cn/textFC/xn.shtml',
            'http://www.weather.com.cn/textFC/gat.shtml']
    for url in urls:
        html = get_html(url)
        if html == 0:
            html = get_html(url)
        parse_html(html)


if __name__ == '__main__':
        main()    

。。港澳台的格式不太一样，暂时不想管他们了

。。运行结果的话，我爬取得是当日白天的天气，现在晚上了，数据都没有了（一开始还以为是代码改错了，还一直撤销）