Python 爬虫-下载图片

运行系统：MAC

Python版本：python3.5

主要模块：Requests、BeautifulSoup

下载网站Unsplash中的图片：

程序结构图如下：

代码如下：

import requests
import os
from bs4 import BeautifulSoup

class Picture():

    def __init__(self):
        # 给请求指定一个请求头来模拟chrome浏览器
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36'}

        self.web_url = 'https://unsplash.com/'
        self.base_path = os.path.dirname(__file__)

    def makedir(self, name):
        path = os.path.join(self.base_path, name)
        isExist = os.path.exists(path)
        if not isExist:
            os.makedirs(path)
            print('File has been created.')
        else:
            print('The file is existed.')
        #切换到该目录下
        os.chdir(path)

    def request(self, url):
        r = requests.get(url, headers=self.headers)
        return r

    def get_img(self):
        r = self.request(self.web_url)
        # r.text是返回的网页HTML;这里的find_all('a', class_='cV68d') 是找到所有class为cV68d的a标签，返回的是一个list，所以可以用for循环获取每个a标签
        all_a = BeautifulSoup(r.text, 'lxml').find_all('a', class_='cV68d')
        self.makedir('pictures')
        for a in all_a:
            # a标签中完整的style字符串
            img_str = a['style']
            # 使用python的切片功能街区双引号之间的内容
            img_url = img_str[img_str.index('"') + 1: img_str.index('"', img_str.index('"') + 1)]
            start_pos = img_url.index('photo')
            end_pos = img_url.index('?')
            file_name = img_url[start_pos: end_pos] + '.jpg'
            with open(file_name, 'ab') as f:
                r = requests.get(img_url)
                f.write(r.content)

if __name__ == '__main__':
    picture = Picture()
    picture.get_img()