爬虫系列之requests

爬取百度内容：

import requests
url = "https://www.baidu.com"

if __name__ == '__main__':
    try:
        kv = {'user-agent': 'Mozilla/5.0'}
        r = requests.get(url, headers=kv)
        r.raise_for_status()  #返回状态值，如果不是200，则抛出异常
        r.encoding = r.apparent_encoding 
        print(r.text)
        #print(r.request.headers)
    except:
        print("爬虫失败")

在URL中填上http://www.baidu.com/s?wd=keyword，keyword就是我们要百度搜索的内容，在requests中有params参数，可以把参数追加到URL中。

import requests
url = "http://www.baidu.com/s"
keyword = "python"

if __name__ == '__main__':
    try:
        kv = {'user-agent': 'Mozilla/5.0'}
        wd = {'wd': keyword}
        r = requests.get(url, headers=kv, params=wd)
        print(r.request.url)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        print(len(r.text))
    except:
        print("爬虫失败")

爬取图片

import requests
import os
url = "http://image.nationalgeographic.com.cn/2017/0211/20170211061910157.jpg"

kv = {'header': 'Mozilla/5.0'}
root = "D://pic_save//"
path = root + url.split('/')[-1]

if __name__ == '__main__':
    try:
        if not os.path.exists(root):
            os.mkdir(root)
        if not os.path.exists(path):
            r = requests.get(url, headers=kv)
            print(r.status_code)
            with open (path, 'wb') as f:
                f.write(r.content)
                print("文件已保存成功")
        else:
            print("文件已存在")
    except:
        ("爬虫失败")