import re import requests url = 'http://news.baidu.com/' headers = { "User-Agent": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36' } # response.text 不太准确 转码 是靠推测 data = requests.get(url, headers=headers).content.decode() # 正则解析 数据 # 每个新闻的titile, url # <a href="http://news.cnr.cn/native/gd/20181028/t20181028_524397644.shtml" target="_blank" mon="r=1">民营经济再吃定心丸,民企当体会怎样深意</a> pattern = re.compile('<a href="(.*?)" target="_blank" mon="(.*?)">(.*?)</a>') # pattern = re.compile('<a (.*?)</a>',re.S) result = pattern.findall(data) print(result) # with open('02news.html', 'w') as f: # f.write(data)