爬虫day1

常用模块  requests   BeautifulSoup
1 汽车之家爬虫练习

import requests
from bs4 import BeautifulSoup
ret = requests.get(url="https://www.autohome.com.cn/news/")
ret.encoding = ret.apparent_encoding
# print(ret.text)
soup = BeautifulSoup(ret.text,'html.parser')
div = soup.find(name='div',id='auto-channel-lazyload-article')
li_list = div.find_all(name='li')
for li in li_list:
    h3 = li.find(name='h3')
    if not h3:
        continue
    # print(h3.text)

    p = li.find(name='p')
    # print(p.text)

    a = li.find(name='a')
    # print(a.get('href'))

    img = li.find('img')
    src = img.get('src')

    file_name = src.rsplit('__',1)[1]
    print(file_name)
    ret_img = requests.get(
        url= 'https:'+src
    )
    with open(file_name,'wb') as f:
        f.write(ret_img.content)

2 抽屉登陆点赞练习

import requests
from bs4 import BeautifulSoup
# 先访问页面，返回cookie
r1 = requests.get(url='https://dig.chouti.com/all/new/1',
                headers={
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36)"})

r1_cookie_dict = r1.cookies.get_dict()


# 登录请求，参数的设置
response_login = requests.post(url='https://dig.chouti.com/login',
                    data={
                        "phone":"8618387391326",
                        "password":"zmq251010",
                        'oneMonth':'1'    },
                    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36)"},
                    cookies = r1_cookie_dict
                               )
print(response_login.text)

for i in range(3,5):
    response_index = requests.get(url='https://dig.chouti.com/all/new/%s' %i,
                 headers={
                     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36)"})
    soup = BeautifulSoup(response_index.text,'html.parser')
    div = soup.find(attrs={'id':'content-list'})
    items = div.find_all(attrs={'class':'item'})
    for item in items:
        tag = item.find(attrs={'class':'part2'})
        nid = tag.get('share-linkid')


        # 根据每个新点赞操作
        ret = requests.post(url='https://dig.chouti.com/link/vote?linksId=%s' %nid,
                            headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36)"},
                            cookies =r1_cookie_dict )
        print(ret.text)

request常用参数
import requests


requests.get(
    url='x',
    params={'nid':1,'name':'x'},#x?nid=1&name=x
    header={},
    cookies={}
)


requests.post(
    url='x',
    data={
        'name':'alex',
        'age':18
    },
    header= {},
    cookie = {},
)

request参数
method:
url:
params:
data:
json:
headers:
cookies:
files:上传文件