day111 爬虫第一天

一、模拟浏览器发请求.

 

import requests
r1 =requests.get(
    url ="https://dig.chouti.com/",
    headers ={
        "user-agent":'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36' # 模拟浏览器
            }
)
print(r1.text)  

　　

二、拿到访问的cookie  (cookie.get_dict)

import requests
r1 =requests.get(
    url ="https://dig.chouti.com/",
    headers ={
        "user-agent":'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
            }
                )
r1_cookie_dict =r1.cookies.get_dict()  #取cookie方式.
print(r1_cookie_dict)

三 、 通过拿到的Cookie自动登录

import requests
r1 =requests.get(
    url ="https://dig.chouti.com/",
    headers ={
        "user-agent":'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
            }
                )
r1_cookie_dict =r1.cookies.get_dict()
print(r1_cookie_dict)
打印cookie 数据{'gpsd': '2b374387cb18e6231dad05778939ed9e', 'JSESSIONID': 'aaaq8zR3Ff_WQ8XSSeysw'}

import requests
r2 =requests.post(
    url= 'https://dig.chouti.com/login',
    headers={
        'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    }, # headers 里的数据为请求头.
    data={
        "phone":"8618611998441",
        "password":"xxx",
        "oneMonth":1
    },  #data 里的数据为请求体.
    cookies =r1_cookie_dict  #通过第一次访问拿到cookie
)
print(r2.text) #打印请求结果

打印结果：{"result":{"code":"9999", "message":"", "data":{"complateReg":"0","destJid":"cdu_53188065757"}}}

　　

 四、点赞请求

r3 =requests.post(
    url="https://dig.chouti.com/link/vote?linksId=20889331",
    headers={
        'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    },
    cookies =r1_cookie_dict
)
print( "r3.text===>",r3.text)

打印结果：r3.text===> {"result":{"code":"30010", "message":"你已经推荐过了", "data":""}}

 总结 （三步骤）

#第一步 拿到cookie

import requests
r1 =requests.get(
    url ="https://dig.chouti.com/",
    headers ={
        "user-agent":'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
            }
                )
r1_cookie_dict =r1.cookies.get_dict()
print("r1_cookie====>",r1_cookie_dict)


#第二步登录

import requests
r2 =requests.post(
    url= 'https://dig.chouti.com/login',
    headers={
        'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    },
    data={
        "phone":"8618611998441",
        "password":"xxx",
        "oneMonth":1
    },
    cookies =r1_cookie_dict
)
print("r2.text===>",r2.text)


#第三步点赞

r3 =requests.post(
    url="https://dig.chouti.com/link/vote?linksId=20889331",
    headers={
        'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    },
    cookies =r1_cookie_dict
)
print( "r3.text===>",r3.text)

作业

作业：

1. 爬取抽屉新热榜的新闻：

标题
简介
地址
图片

2. 煎蛋网
- 爬取标题+简介
- 爬取图片

 

 

 

 一 、 抽屉网站爬虫

import os
import  requests
from bs4 import  BeautifulSoup


#1. 伪造浏览器发送请求
r1 =requests.get(
    url = "https://www.autohome.com.cn/news/"
)
r1.encoding="gbk"
print(r1.text)

#2.去响应 的响应体中解析我们想要的数据.
soup =BeautifulSoup(r1.text,"html.parser")

#3. 找名字按照响应的规则：div 标签且 id = auto -channel-lazyload-article找匹配成功的第一个
container =soup.find(name="div",attrs={"id":"auto-channel-lazyload-article"})

#4.去container中找所有的li标签
li_list =container.find_all(name ="li")
for tag in li_list:
    title =tag.find(name ="h3")
    if not title:
        continue
    summary =tag.find(name="p")
    a =tag.find(name="a")
    url ="https:"+a.attrs.get("href")

    img= tag.find(name="img")
    img_url= "https:"+img.get("src")
    print(title.text)
    print(summary.text)
    print(url)
    print((img_url))

#下载图片
r2 =requests.get(
    url=img_url
)
file_name =img_url.rsplit("/",maxsplit=1)[1]
file_path=os.path.join("imgs",file_name)
with open(file_path,"wb")as f:
    f.write(r2.content)

　　

"""
作业：
	1. 爬取抽屉新热榜的新闻：
		标题
		简介
		地址
		图片
	2. 煎蛋网
		- 爬取标题+简介
		- 爬取图片


"""


import requests
from bs4 import BeautifulSoup

#1. 伪造浏览器发送请求
r1 =requests.get(
    url="https://dig.chouti.com",
    headers={
    "user-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36"
    }
)

#2.去响应的响应体重解析我们想要的数据
soup =BeautifulSoup(r1.text,"html.parser")
container = soup.find(name ="div",attrs={"id":"content-list"})
div_list = container.find_all(name ="div",attrs = {"class":"part1"})

# 1 拿到标题
# for  item  in  div_list:
#     title = item.find(name ="a")
#     title =title.text
#     title =title.strip()
#     print(title)

#2 拿到简介

# div_list = container.find_all(name ="div",attrs = {"class":"area-summary"})
# for item in div_list:
#     summary = item.find(name ="span",attrs ={"class":"summary"})
#     print(summary,type(summary))

#3.拿到地址:

# for item in div_list:
#     tag =item.find(name ="a",attrs = {"class":"show-content color-chag"})
#     url=tag.attrs.get("href")
#     print(url)

#4. 图片.

div_item =container.find_all(name ="div",attrs ={"class":"item"})
for item in div_item:
    div_pic = item.find(name="div", attrs={"class": "news-pic"})
    print(div_pic)
    pic =div_pic.find("img")
    img_url ="https://"+pic.get("original")  #图片的url
    print(img_url)

　　

二 、煎蛋网爬虫 

import requests
from bs4 import BeautifulSoup

r1 =requests.get(
    url ="http://jandan.net"
)
soup =BeautifulSoup(r1.text,"html.parser")
container = soup.find(name ="div",attrs={"id":"content"})
div_list = container.find_all( name ="div",attrs={"class": "post f list-post"})



#1 打印出所有的标题.
# for item in div_list:
#     div_index =item.find(name ="div",attrs ={"class":"indexs"})
#     title = div_index.find(name ="h2")
#     title =title.find(name="a")
#     print(title.text)


#2 .打印出所有的简介.

for item in div_list:
    div_index =item.find(name ="div",attrs ={"class":"indexs"})
    # print(len(div_index.contents))
    print(div_index.contents[6])#共计7个长度，标签之间空格也算一个。