电影天堂爬取详情页

爬取电影天堂最新电影,地址https://www.dytt8.net/html/gndy/dyzz/list_23_1.html

import requests
from lxml import etree

BASE_DOMAIN = 'https://www.dytt8.net'
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}


def get_detail_urls(url):
    response = requests.get(url, headers=HEADERS)
    text = response.content.decode(encoding='gbk', errors='ignore')
    html = etree.HTML(text)
    detail_urls = html.xpath('//table[@class="tbspan"]//a/@href')
    detail_urls = map(lambda url: BASE_DOMAIN+url, detail_urls)
    return detail_urls


def get_detail_info(url):
    movie = {}
    response = requests.get(url, headers=HEADERS)
    text = response.content.decode(encoding='gbk', errors='ignore')
    html = etree.HTML(text)
    movie['title'] = html.xpath('//div[@class="title_all"]//font/text()')[0]
    img = html.xpath("//div[@id='Zoom']//img/@src")
    movie['cover'] = img[0]
    movie['screenshot'] = img[1]
    infos = html.xpath('//div[@id="Zoom"]//p/text()')
    # 提取信息
    is_actors = False
    actors = []
    for info in infos:
        # print(info)
        if info.startswith('◎年　　代'):
            movie['year'] = info.replace("◎年　　代", "").strip()
        elif info.startswith('◎产　　地'):
            movie['country'] = info.replace("◎产　　地", "").strip()
        elif info.startswith('◎类　　别'):
            movie['category'] = info.replace("◎类　　别", "").strip()
        elif info.startswith('◎豆瓣评分'):
            movie['douban_rating'] = info.replace("◎豆瓣评分", "").strip()
        elif info.startswith('◎片　　长'):
            movie['duration'] = info.replace("◎片　　长", "").strip()
        elif info.startswith('◎导　　演'):
            movie['director'] = info.replace("◎导　　演", "").strip()
        elif info.startswith('◎主　　演'):
            actors = [info.replace("◎主　　演", "").strip()]
            is_actors = True
        elif is_actors:
            if info.startswith('◎'):
                is_actors = False
                movie['actors'] = actors
                continue
            actors.append(info.strip())
    movie['download'] = html.xpath("//div[@id='Zoom']//tbody//a/text()")
    movie['magnet'] = html.xpath("//div[@id='Zoom']//a/@href")[0]

    return movie


def spider():
    base_url = 'https://www.dytt8.net/html/gndy/dyzz/list_23_{}.html'
    movies = []
    for i in range(1, 2):
        url = base_url.format(i)
        detail_urls = get_detail_urls(url)
        for detail_url in detail_urls:
            # 对详情页提取信息
            # print(detail_url)
            movies.append(get_detail_info(detail_url))
    print(movies)


if __name__ == '__main__':
    spider()

学习的视频中代码有几处跟我的有不同,可以学习

一、

其中提取主演的代码不同,如下

for index,info in enumerate(infos):
        if info.startswith("◎年　　代"):
            info = parse_info(info,"◎年　　代")
            movie['year'] = info
        # .......省略
        elif info.startswith("◎主　　演"):
            info = parse_info(info,"◎主　　演")
            actors = [info]
            for x in range(index+1,len(infos)):
                actor = infos[x].strip()
                if actor.startswith("◎"):
                    break
                actors.append(actor)
            movie['actors'] = actors

采用的是index的方式.

enumerate() 函数用于将一个可遍历的数据对象(如列表、元组或字符串)组合为一个索引序列，同时列出数据和数据下标，一般用在 for 循环当中。

二、

还有

detail_urls = map(lambda url:BASE_DOMAIN+url,detail_urls)

这段代码没怎么用过.记录下.

三、

base_url = 'https://www.dytt8.net/html/gndy/dyzz/list_23_{}.html'
    for i in range(1, 10):
        url = base_url.format(i)

以前写的时候没这么写过，都是直接弄成

url = 'https://www.dytt8.net/html/gndy/dyzz/list_23_'+i+'.html'这样.

四、

在自己写代码时有个错误,也需要记录下,在movie的字典赋值的时候,

如下

    for info in infos:
        # print(info)
        if info.startswith('◎年　　代'):
            year = info.replace("◎年　　代", "").strip()
        elif info.startswith('◎产　　地'):
            country = info.replace("◎产　　地", "").strip()
        elif info.startswith('◎类　　别'):
            category = info.replace("◎类　　别", "").strip()
        elif info.startswith('◎豆瓣评分'):
            douban_rating = info.replace("◎豆瓣评分", "").strip()

赋值的时候使用

 movie = {
        'year': year,
        'country': country,
        'category': category,
        'douban_rating': douban_rating
    }

会报错,因为其中有一个豆瓣评分是不存在的,不会对其赋值,所以movie赋值的时候会错误.

UnboundLocalError: local variable 'douban_rating' referenced before assignment

 当然，个人觉得用正则可以更容易解决。

在记录下爬取豆瓣正在上映的电影用xpath的代码

import requests
from lxml import etree

headers = {
    'Referer': 'https://movie.douban.com/',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}
url = "https://movie.douban.com/cinema/nowplaying/changsha/"
response = requests.get(url, headers=headers)
text = response.text
html = etree.HTML(text)
lis = html.xpath("//div[@id='nowplaying']//ul[@class='lists']/li")
movies = []
for li in lis:
    # print(etree.tostring(li, encoding='utf-8').decode('utf-8'))
    title = li.xpath("@data-title")[0]
    score = li.xpath("@data-score")[0]
    duration = li.xpath("@data-duration")[0]
    director = li.xpath("@data-director")[0]
    actor = li.xpath("@data-actors")[0]
    img = li.xpath(".//img/@src")[0]
    movie = {
        'title': title,
        'score': score,
        'duration': duration,
        'director': director,
        'actor': actor,
        'img': img
    }
    movies.append(movie)

print(movies)