使用requests模块简单获取数据

一、使用ruquests的基本步骤：

• 指定url
• 发起请求
• 获取响应对象中的数据
• 持久化存储

#1
url = 'https://www.sogou.com/'
#2.
response = requests.get(url=url)
#3.
page_text = response.text
#4.
with open('./sogou.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

二、爬取搜狗指定搜索

import requests
url = "'https://www.sogou.com/web"
wd = input("请输入搜索关键字")
param = {
    'query':wd
}

response = requests.get(url=url,params=param).content
filename = wd+'.html'
with open(filename,'w',encoding='utf8') as f1:
        f1.write(response)

三、Ajax请求

通过抓包，获取请求携带的参数，

例如获取分页显示的数据，当点击下一页时，发送ajax请求，对此时的url请求可以动，这里我们定义好请求参数param，动态的指定页码和每页显示的数据，通过ajax请求，返回一组json数据

存储每页获取的数据的id，编辑new_url，获取详情的信息

import requests
url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsList'
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0'
}
param = {
   "on":"true",
    "page":1,
    "pageSize":"15",
    "productName":"",
    "conditionType":"1",
    "applyname":"",
    "applysn":"",
}
id_list = []
json_object = requests.post(url=url,headers=headers,params=param).json()
print(json_object['list'])
for i in json_object['list']:
    id_list.append(i['ID'])
    
new_url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsById'
filename = 'yaojians.text'
f = open(filename,'w',encoding='utf8')
for id in id_list:
    param = {
        'id':id
    }
    content_json = requests.post(url=new_url,params=param,headers=headers).json()
    f.write(str(content_json)+'
')
    
    
    

四、使用BeautifullSoup爬取数据

bs4解析：
　　pip install bs4
　　pip install lxml

解析原理
　　1、将要进行解析的源码加载到bs对象

　　2、调用bs对象中相关的方法或属性进行源码中的相关标签的定位

　　3、将定位到的标签之间存在的文本或属性值获取

import requests
from bs4 import BeautifulSoup

url = 'http://www.shicimingju.com/book/sanguoyanyi.html'
headers = {
    "User-Agent":'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
res = requests.get(url=url, headers=headers).text
soup = BeautifulSoup(res, 'lxml')
a_tags_list = soup.select('.book-mulu > ul > li > a')
filename = 'snaguo.text'
fp = open(filename, 'w', encoding='utf-8')
for a_tag in a_list:
    title = a_tag.string
    detail_url = "http://www.shicimingju.com"+a_tag["href"] 
    detail_content = requests.get(url=detail_url, headers=headers).text
    soup = BeautifulSoup(detail_content, "lxml")
    detail_text = soup.find('div', class_="chapter_content").text
    fp.write(title+'
'+detail_text)
    print(title, '下载完毕')
print('over')
fp.close()
    

五、简单使正则爬取图片

url = 'https://www.qiushibaike.com/pic/page/%d/?s=5170552'
start_page = int(input("请输入起始页:"))
end_page = int(input("请输入结束页:"))
headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0"
}
for page in range(start_page,end_page+1):
    new_url = format(url%page)
    response = requests.get(url=new_url, headers=headers).text
    # 每一页的图片url
    images_url = re.findall('<div class="thumb">.*?<img src="(.*?)" alt=.*?</div>',response,re.S)
os.mkdir('qiutu')
    
    for image_url in images_url:
        detail_url = 'http:'+image_url
        # 获取到当前图片的二进制流
        content = requests.get(url=detail_url,headers=headers).content
        # 切割 把图片路径最后的字符作为图片名
        image_name = image_url.split('/')[-1]
        with open('./qiutu/'+image_name,'wb')as f1:
            f1.write(content)
print('over')