python爬虫-爬取你想要的小姐姐

一、准备

1. 原地址

2. 检查html发现，网页是有规则的分页, 最大图片的class为pic-large

二、代码

import requests
import os
from bs4 import BeautifulSoup

url = 'http://www.win4000.com/wallpaper_detail_157712.html'
imgmkdir = 'D://Download//ghost_1//'


# 获取网页url
def getUrlList():
    imgUrlList = []
    for i in range(0, 10):
        imgUrl = ''
        url_split = url.split('.html')
        if not i == 0:
            imgUrl += url_split[0] + '_' + str(i) + '.html'
            # print(imgUrl)
            imgUrlList.append(imgUrl)

    return imgUrlList


# 下载图片
def downImg(imgUrl):
    try:
        if not os.path.exists(imgmkdir):
            os.mkdir(imgmkdir)
        if not os.path.exists(imgUrl):
            r = requests.get(imgUrl)
            r.raise_for_status()
            # 使用with语句可以不用自己手动关闭已经打开的文件流
            imgpath = imgmkdir + imgUrl.split('/')[-1]
            # 开始写文件, wb表示写二进制文件
            with open(imgpath, 'wb') as f:
                f.write(r.content)
            print(imgUrl + '【爬取完成】')
        else:
            print(imgUrl.split('/')[-1] + '【文件已存在】')
    except Exception as e:
        print("爬取失败" + str(e))


# 获取imgHtml标签
def getcontent(soup):
    for i in soup.find_all('img', class_='pic-large'):
        imgsrc = i['src']
        if imgsrc.find('http') >= 0 or imgsrc.find('https') >= 0:
            # 下载图片
            downImg(imgsrc)


# 根据url获取html源码
def getHtmlByUrl(htmlUrl):
    htmlText = requests.get(htmlUrl).content
    # 使用beautifulSoup解析html
    soup = BeautifulSoup(htmlText, 'lxml')

    return soup


def main():
    htmlUrlList = getUrlList()
    for url in htmlUrlList:
        htmltext = getHtmlByUrl(url)
        getcontent(htmltext)


if __name__ == '__main__':
    main()

三、结果

四、总结

　　代码用比较笨的方法来获取，先试水