简单爬虫一部美剧（二：下载进度条）

上篇获取到了每一集的下载url，这篇就用获取到的url下载视频

在下载文件时希望可以给出进度条之类的提示，在网上搜索了一波，发现有一个库可以实现：tqdm库，具体用法参考这篇博客：https://www.jianshu.com/p/1ed2a8b2c77b

在原来的类下面，再加一个方法，用来下载文件，如下

    def download_file(url, name):
        """下载文件"""
        try:
            response = requests.get(url=url, stream=True)
            content_size = int(response.headers['Content-Length']) / 1024  # 文件大小，从响应头中获取

            if content_size:
                with open(name, "wb") as f:
                    print("total: ", content_size, 'k')
                    for data in tqdm(iterable=response.iter_content(1024), total=content_size, unit='k'):  # 实现下载进度条
                        f.write(data)
                        # f.close()
                    print("
 done " + name)

        except RequestException as e:
            print("接口错误信息为  %r", e)

原有基础上加了一个主函数

    def main(self):
        """主函数"""
        root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
        print(root_dir)
        download_url = self.get_tv_url("血疫第一季")  # 调用get_tv_url()，获取剧集url
        for t in download_url:
            name = t.split('/')[-1] #　获取下载链接url中/后的一段文本，用作文件名
            file_path = root_dir + "/movies/"  #　设置文件保存路径
            if not os.path.exists(file_path):
                os.makedirs(file_path)
            print("正在下载【{}】".format(name))
            self.download_file(t, file_path+name)  # 调用download_file()，根据下载链接开始下载文件

最后完整的代码如下

# coding: utf-8
"""
author: 我是冰霜
describe: 爬虫80s电影网
create_time: 2020/01/18
"""

import re

import os
from bs4 import BeautifulSoup
from requests.exceptions import RequestException
import requests
from tqdm import tqdm

class DownloadTV:
    @staticmethod
    def get_html(url, data=None, header=None, method=None):
        """获取一个url的html格式文本内容"""

        if method == "get":
            response = requests.get(url, params=data, headers=header)
        else:
            response = requests.post(url, data=data, headers=header)
        try:
            if response.status_code == 200:
                response.encoding = response.apparent_encoding
                # print(response.status_code)
                # print(response.text)
                return response.text
            return None
        except RequestException:
            print("请求失败")
            return None

    def get_tv_id(self, tv_name):
        """获取所查询电视剧对应的id"""
        headers = {
            "Content-Type": "application/x-www-form-urlencoded"
        }

        data = {
            "search_typeid": "1",
            "skey": tv_name,  # 用一个变量来表示要搜索的剧名
            "Input": "搜索"
        }

        url = "http://www.y80s.com/movie/search/"  # 请求url

        response = self.get_html(url, data, headers, "post")

        html = response
        # print(html)

        soup = BeautifulSoup(html, "html.parser")
        name_label = soup.find_all("a", title=tv_name)  # 获取所有title属性为影视剧名称的<a>标签,用一个变量来动态表示剧名
        # print(soup.prettify())
        # print(name_label)
        # print(name_label[0].get('href'))

        ju_id = re.compile(r'(d+)', re.S)  #  定义一个正则表达式，提取标签内容中的数字
        if name_label:
            href_value = ju_id.search(name_label[0].get('href'))
            if href_value:
                tv_id = href_value.group()
                print("查询影视剧对应的id为：{}".format(tv_id))
                # print(type(tv_id))  # 查看获取到的tv_id的数据类型，如果是int的话，在后续拼接时需要使用str()转成字符串
                return tv_id

    def get_tv_url(self, tv_name):
        """获取电视剧的下载url"""
        tv_id = self.get_tv_id(tv_name) # 调用get_tv_id()方法，获取tv_id
        url = "http://www.y80s.com/ju/" + tv_id  # 利用tv_id拼接url

        r = self.get_html(url,method="get")
        html = r
        soup = BeautifulSoup(html, "html.parser")
        a_tv_url = soup.find_all("a", title="本地下载")  # 提取title属性为"本地下载"的a标签，返回一个包含所有a标签的列表
        # print(a_tv_url)
        tv_url = []
        for t in a_tv_url:
            tv_url.append(t.get('href'))  # 用get方法获取每个a标签中的href属性值
        print(tv_url)
        return tv_url

    @staticmethod
    def download_file(url, name):
        """下载文件"""
        try:
            response = requests.get(url=url, stream=True)
            content_size = int(response.headers['Content-Length']) / 1024  # 文件大小，从响应头中获取

            if content_size:
                with open(name, "wb") as f:
                    print("total: ", content_size, 'k')
                    for data in tqdm(iterable=response.iter_content(1024), total=content_size, unit='k'):
                        f.write(data)
                        # f.close()
                    print("
 done " + name)

        except RequestException as e:
            print("接口错误信息为  %r", e)

    def main(self):
        """主函数"""
        root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
        print(root_dir)
        download_url = self.get_tv_url("血疫第一季")  # 调用get_tv_url()，获取剧集url
        for t in download_url:
            name = t.split('/')[-1] #　获取下载链接url中/后的一段文本，用作文件名
            file_path = root_dir + "/movies/"  #　设置文件保存路径
            if not os.path.exists(file_path):
                os.makedirs(file_path)
            print("正在下载【{}】".format(name))
            self.download_file(t, file_path+name)  # 调用download_file()，根据下载链接开始下载文件

if __name__ == '__main__':
    test = DownloadTV()
    test.main()

运行一下