requests库的基础知识

1.安装。

cmd----------->> pip install requests.

2. 七种操作方法。

# GET        全部信息
# HEADER     仅头部信息

# Put        全部重置 
# Patch      局部更新              ## 更改操作用户
# POST       后面添加新内容         ## 搜索使用
# DELETE     删除全部

import requests

''' 
r = requests.get("http://www.baidu.com")      # 获得全部文本信息 uRL对应的页面内容
print(r.headers)                              # 头部信息
print(r.text)                                 # seem is also all information
'''

# requests.head
'''
r2 = requests.head("http://www.baidu.com")       # just head information
print(r.headers)                                 # head information
print(r2.text)                                   # no ! because just get the head information
'''

#
payload = {"key1":"value1","key2":"value2"}
r3 = requests.post("http://www.baidu.com",data=payload)
print(r3.text)

2.Response对象的属性。

import requests
r = requests.get("http://www.baidu.com")

print(r.status_code)           # HTTP请求的返回状态，200表示连接成功。404表示失败
print(r.text)                  # HTTP响应内容的字符串形式，即，uRL对应的页面内容。
print(r.encoding)              # 从HTTP header中猜测的响应内容编码方式
print(r.apparent_encoding)     # 内容中分析出的响应内容编码方式
print(r.content)               # 响应内容的二进制形式 （处理图片，视频等使用）

r.encoding = r.apparent_encoding # 转化编码   r.apparent_encoding  根据它的结果转码
print(r.text)

小结：通过 r.status_code 返回的状态码，判断是否连接成功。

3.通用代码框架。

def getHTMLText(url):
    try:
        r = requests.get(url,timeout = 30)
        r.raise_for_status()   # 如果状态是200，引发异常
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "404"

if __name__ == "__main__":      # 没搞懂这个是什么鬼
    url = "http://www.baidu.com"
    print(getHTMLText(url))

4.ROBOTS.txt协议。

实战练习。

1.京东页面的提取。

'''
import requests
r = requests.get('https://item.jd.com/13115733485.html')
print(r.status_code)
print(r.encoding)
print(r.text)
'''

'''
import requests
url = 'https://item.jd.com/13115733485.html'
try:
    r = requests.get(url, timeout=30)
    r.raise_for_status()
    r.encoding = r.apparent_encoding
    print(r.text[:1000]) 
except:
    print("404")
'''

import requests
def getHTMLText(url):
    try:
        r = requests.get(url,timeout = 30)
        r.raise_for_status()   # 如果状态是200，引发异常
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "404"

if __name__ == "__main__":
    url = "https://item.jd.com/13115733485.html"
    print(getHTMLText(url)[:1000])

2.亚马逊。

'''
import requests
r = requests.get("https://www.amazon.cn/gp/product/B00PG0MMLO/ref=s9_acsd_al_bw_c_x_5_w?pf_rd_m=A1AJ19PSB66TGU&pf_rd_s=merchandised-search-5&pf_rd_r=N8WFJMBB60D92VPHAREM&pf_rd_r=N8WFJMBB60D92VPHAREM&pf_rd_t=101&pf_rd_p=26451395-7952-4f3c-b948-09e79ff542f8&pf_rd_p=26451395-7952-4f3c-b948-09e79ff542f8&pf_rd_i=1885051071")
print(r.status_code)                     # 503   连接出错，不是404
print(r.encoding)                        # ISO-8859-1
r.encoding = r.apparent_encoding         # 转码
print(r.text)           # 有反应，说明受限  # 报歉，由于程序执行时，遇到意外错误，您刚刚操作没有执行成功，请稍后重试。或将此错误报告给我们的客服中心

print(r.request.headers)   # 获取请求头部信息     # {'User-Agent': 'python-requests/2.14.2', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}
'''

'''
import requests
kv = {'User-Agent':"Mazilla/5.0"}
url = "https://www.amazon.cn/gp/product/B00PG0MMLO/ref=s9_acsd_al_bw_c_x_5_w?pf_rd_m=A1AJ19PSB66TGU&pf_rd_s=merchandised-search-5&pf_rd_r=N8WFJMBB60D92VPHAREM&pf_rd_r=N8WFJMBB60D92VPHAREM&pf_rd_t=101&pf_rd_p=26451395-7952-4f3c-b948-09e79ff542f8&pf_rd_p=26451395-7952-4f3c-b948-09e79ff542f8&pf_rd_i=1885051071"

r = requests.get(url,headers= kv)       # 更改头部信息
print(r.status_code)                    # 200
print(r.request.headers)                # {'User-Agent': 'Mazilla/5.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}
print(r.text[:1000])
'''

import requests

def getHTMLText(url):
    try:
        kv = {'User-Agent': "Mazilla/5.0"}
        r = requests.get(url,headers= kv,timeout = 30)         # headers=
        r.raise_for_status()   # 如果状态是200，引发异常
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "404"

if __name__ == "__main__":
    url = "https://www.amazon.cn/gp/product/B00PG0MMLO/ref=s9_acsd_al_bw_c_x_5_w?pf_rd_m=A1AJ19PSB66TGU&pf_rd_s=merchandised-search-5&pf_rd_r=N8WFJMBB60D92VPHAREM&pf_rd_r=N8WFJMBB60D92VPHAREM&pf_rd_t=101&pf_rd_p=26451395-7952-4f3c-b948-09e79ff542f8&pf_rd_p=26451395-7952-4f3c-b948-09e79ff542f8&pf_rd_i=1885051071"
    print(getHTMLText(url)[:1000])

3.百度搜索

'''
import requests
kv = {"wd":"Python"}
r = requests.get("http://www.baidu.com/s",params=kv)
print(r.status_code)                  # 200
print(r.request.url)                  # http://www.baidu.com/s?wd=Python
print(len(r.text))                    # 196429
'''
import requests
keyword = "Python"
def getHTMLText(url):
    try:
        kv = {'wd':keyword}                                      # 如何添加的？？？？
        r = requests.get(url,params=kv,timeout = 30)              # params
        r.raise_for_status()   # 如果状态是200，引发异常
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "404"

if __name__ == "__main__":
    url = "http://www.baidu.com"
    print(getHTMLText(url)[:1000])

其他控制参数的使用方法：

实例 查询IP

import requests
url = "http://www.ip138.com/ips138.asp?ip="
r = requests.get(url+"202.204.80.112")           # 对URL内容进行修改
print(r.status_code)
print(r.encoding)
# r.encoding = "utf-8"
print(r.text[:-500])

下载图片：

'''
import requests
path = "F:/abc.jpg"
url = "http://image.nationalgeographic.com.cn/2017/0721/20170721020325584.jpg"
r = requests.get(url)
print(r.status_code)

with open(path,"wb") as f:         # 文件存储
    f.write(r.content)   # 文件写入          # r.content 响应内容的二进制形式  # 这个句子是什么意思来着？？

'''

import requests
import os

url = "http://image.nationalgeographic.com.cn/2017/0721/20170721020325584.jpg"
root = 'F://pics//'
path = root + url.split('/')[-1]

if not os.path.exists(root):
    os.mkdir(root)
if not os.path.exists(path):
    r = requests.get(url)
    with open(path, 'wb') as f:
        f.write(r.content)
        f.close()
        print('文件保存成功！')
else:
    print('文件已存在。')

遇到问题很多，需要多多练习呀！！！