【爬虫】Python3搜索并获取电影网站的迅雷下载链接

用到的python第三方库：bs4、requests、execjs（在python中调用js脚本）

目标网站: www.beiwo.tv

通过分析网页源码发现，网页通过调用一个加密JS脚本来加密网页源码中的一段字符串从而获得迅雷电影链接，将分析网页得到的JS加密源码存放到encode.js文件中，通过exejs库在python中调用并传递我们抓取到的待加密的字符串从而获得想要得到的电影迅雷下载链接。

抓包可以分析出网页搜索所发送的表单，构造data表单，向网页发送请求获得搜索网页url，然后按照上面所说解析网页获得待加密字符串。

源码如下：

#ThunderUrl.py
import execjs
def echoDown(str):
    #获得未编码的迅雷链接
    gurl = []
    s = str.split("###")
    for i in range(0,len(s)-1):
        gurl = s[i].split("$")
    return gurl[1]

def getJs():
    #生成JS脚本对象
    f = open("encode.js", 'r')
    line = f.readline()
    htmlstr = ''
    while line:
        htmlstr = htmlstr + line
        line = f.readline()
    return htmlstr

"""
将str编码成可用的迅雷链接
"""
def getThunderUrl(str):
    t_url = echoDown(str)
    jsstr = getJs()
    ctx = execjs.compile(jsstr)
    ThunderUrl = ctx.call('ThunderEncode',t_url)
    print(ThunderUrl)

#MovieSearch.py
import requests
from bs4 import BeautifulSoup
import re

def MovieSeach(movieName):
#根据movideName 从url中请求搜索结果 并将结果url存放到集合movieUrlSet中
    name = movieName
    data = {'typeid': '2', 'wd': name}
    url = 'http://www.beiwo.tv/index.php?s=vod-search'
    headers = {
        'Cache-Control': 'max-age=0',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36'
    }
    response = requests.post(url, data=data)
    if response.status_code != 200:
        print('网页打开失败')
        exit(0)
    response.raise_for_status()
    response.encoding = response.apparent_encoding
    soup = BeautifulSoup(response.text, 'lxml')
    movieUrl = soup.find_all('a')
    movieUrlSet = set()
    for movie in movieUrl:
        temp = re.findall(r'(/vod/.*/)', movie['href'])
        if len(temp):
            url = 'http://www.beiwo.tv' + "".join(list(temp))
            movieUrlSet.add(url)
    return movieUrlSet

"""
解析urls中的url获得编码前的迅雷链接
将迅雷链接存放到Thunders中
"""
def UrlParser(url):
    response = requests.get(url)
    if response.status_code != 200:
        print('网页解析失败')
        exit(0)
    response.raise_for_status()
    response.encoding = response.apparent_encoding
    soup = BeautifulSoup(response.text,'lxml')
    ThunderUrls = soup.find_all('script')
    Thunders = re.findall(r'var GvodUrls3 = "(.*###)";',str(ThunderUrls))
    return Thunders

#main.py
from ThunderUrl import *
from MovieSearch import *

if __name__ == "__main__":
    print('请输入要搜索的影片名:',end="")
    name = input()
    UrlSet = MovieSeach(name)
    for url in UrlSet:
        Thunders = UrlParser(url)
        for Thunder in Thunders:
            getThunderUrl(Thunder)

运行结果：

 

侵删