电影天堂爬虫实战

#!/usr/bin/python3
# -*- coding:utf-8 -*-
# @Time:2021/8/28 22:38
# @author: Mrwhite
# @File:电影天堂爬虫.py
# @DESC:
import re
import urllib.request,urllib.error   #制定URL，获取网页数据
import xlwt
from bs4 import BeautifulSoup


def main():
    pass
    #电影天堂url
    baseurl = "https://dy.dytt8.net/index.htm"
    
    #1.爬取主页-电影名称,超链接,更新日期
    #据超链地址打开后,获取导演/主演/豆瓣评分/磁力链接/简介
    datalist = getData(baseurl)

    #2.保存数据excel表,根据分类插入对应sheet页
    saveData(datalist,"电影天堂电影.xls")

#创建正则表达式对象,表示规则(字符串的模式)
findLink = re.compile(r'・[<a href="/html/gndy/.*<a href="(.*?)">')   #影片链接匹配规则
findMovieName = re.compile( r'・[<a href="/html/gndy/.*">(.*?)</a><br/>' ) #匹配电影名称
findUpDateTime = re.compile( r'<td class="inddline" width="15%"><font color="#FF0000">(.*?)</font></td>' ) #匹配更新日期
findDirect = re.compile( r'<br />◎导　　演　(.*?)<br />' ) #匹配导演
findActor = re.compile( r'<br />◎主　　演　(.*?)<br /><br />◎标　　签' ) #匹配演员
findScore = re.compile( r'<br />◎豆瓣评分　(.*?) from' ) #匹配豆瓣评分
findDownloadLink = re.compile( r'<a target="_blank" href="(.*?)">' ) #匹配下载链接
findInfo = re.compile( r'◎简　　介<br /><br />　　(.*?)<br />' ) #相信信息

def getData(baseurl):
    datalist = []
    titles,links,updateTimes,directs,actors,scores,downloadLinks,infos=[],[],[],[],[],[],[],[]
    #1.爬取网页
    html = askURl(baseurl)
    #print(html)
    # 2.解析数据
    soup=BeautifulSoup( html, "html.parser" )
    #nth-child需要替换为nth-of-type
    item = soup.select("div:nth-of-type(2) > div:nth-of-type(1) > div > div > div.co_content8")
    item = str(item)
    #print(item)
    titles = re.findall(findMovieName, item) #正则匹配标题
    #links = f'https://dy.dytt8.net/{re.findall(findLink, html)}'
    linksUnSet = re.findall(findLink, item) #正则匹配超链接并拼接完整路径
    for link in linksUnSet:
        link = f'https://dy.dytt8.net{link}'
        links.append(link)
    updateTimes = re.findall(findUpDateTime,item) #正则匹配更新实际

    #3.循环访问电影子链接获取:导演/主演/豆瓣评分/磁力链接/简介
    for link in links:
        #print(link)
        html=askURl(link)
        #print(html)
        directUnSet = re.findall(findDirect,html) # 正则匹配导演并处理
        if directUnSet==[]:
            directs.append("")
        else:
            direct=directUnSet[0].replace(" ","").replace("&middot;","·")
            directs.append(direct)

        actorsUnset = re.findall(findActor,html)  # 正则匹配主演
        if actorsUnset==[]:
            actors.append("")
        else:
            actorList = actorsUnset[0].replace("&middot;","·").replace("&nbsp;","").replace("u3000","").split("<br />")[0:3]
            actor="/".join( actorList )
            actors.append(actor)

        scoresUnset = re.findall(findScore,html)  # 正则匹配豆瓣评分
        if scoresUnset==[]:
            scores.append("无评分")
        else:
            score=scoresUnset[0].split("/")[0]
            scores.append(score)

        downloadLink = re.findall(findDownloadLink,html)  # 正则匹配磁力链接
        downloadLinks.append(downloadLink)

        infosUnSet = re.findall(findInfo,html) # 正则匹配简介
        if infosUnSet==[]:
            infos.append("")
        else:
            info = infosUnSet[0].replace("&middot;","·").replace("&nbsp;","").replace("&ldquo;","")
            infos.append(info)

    dataList=[titles, updateTimes,directs, actors, scores, downloadLinks, infos]
    #print( len( titles ), len( updateTimes ),len(links), len( directs ), len( actors ), len( scores ), len( downloadLinks ),len( infos ) )
    return dataList

#得到指定一个URL的网页内容
def askURl(url):
    #head={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"}
    #request = urllib.request.Request(url,headers=head)
    request = urllib.request.Request(url)
    try:
        response = urllib.request.urlopen(request)
        html = response.read().decode("gb2312",errors = 'ignore')
        #print(html)
    except urllib.error.URLError as e:
        if hasattr(e,"code"):
            print(e.code)
    return html

#保存数据
def saveData(datalist,savepath):
    print("save......")
    book = xlwt.Workbook(encoding="utf8",style_compression=0)
    sheet = book.add_sheet("from电影天堂",cell_overwrite_ok=True)
    col = ('标题',"更新时间","导演","主演","豆瓣评分","磁力链接","简介")
    try:
        for j in range(7): #i为行,j为列
            sheet.write(0,j,col[j]) #列名
            for i in range(1,len(datalist[0])):
                sheet.write(i,j,datalist[j][i])
                print("datalist的",i,"行",j,"列的数据为:",datalist[j][i],"成功写入")
        book.save(savepath)    #保存
    except Exception as e:
        print("datalist的",i,"行",j,"列的数据为:",datalist[j][i],"写入失败")
        print(e)

if __name__ == "__main__":  #当程序执行时
    #调用函数
    main()
    print("爬取完毕")

展示效果如下:可继续添加下优化爬虫的效率