13 爬取豆瓣电影网电影信息

"""豆瓣电影爬虫"""


import requests
from lxml import etree

# 1、将目标网站上的页面爬取出来
headers = {
    'User-Agent': 'Mozilla/5.0',
}

url = 'https://movie.douban.com/cinema/nowplaying/shangrao/'
#url = 'https://movie.douban.com/'

response = requests.get(url, headers=headers)
text = response.text
#print(response.text)
with open('douban.html', 'w', encoding='utf-8') as fp:
    fp.write(response.content.decode('utf-8'))
# response.text返回的是一个经过解码的字符串，是str(unicode)类型
# response.content返回的是一个原生的字符串是bytes类型，没有经过解码，respose.content.decode('utf-8')解码

# 2、将数据根据一定的规则进行提取
movies = []
html = etree.HTML(text)
ul = html.xpath("//ul[@class='lists']")[0]
lis = ul.xpath("./li")
for li in lis:
    title = li.xpath("@data-title")[0]
    region = li.xpath("@data-region")[0]
    director = li.xpath("@data-director")[0]
    actors = li.xpath("@data-actors")[0]
    duration = li.xpath("@data-duration")[0]
    img = li.xpath(".//img/@src")[0]
    release_date = li.xpath(".//li[@class='release-date']/text()")[0].strip()       # strip()去除两边空格
    #print(release_date)

    movie = {
        'title': title,
        'region': region,
        'director': director,
        'actors': actors,
        'duration': duration,
        'img': img,
        'date': release_date
    }
    movies.append(movie)

for movie in movies:
    print(movie)