豆瓣即将上映电影爬虫作业

https://study.163.com/course/courseLearn.htm?courseId=1005913008#/learn/text?lessonId=1053258283&courseId=1005913008

把豆瓣“即将上映”的电影爬出来。

很简陋的代码，现在的知识只能写这样了，做个记录吧。以后有能力再完善。

# coding=gbk
#  豆瓣即将上映的电影
# https://movie.douban.com/cinema/later/changsha/爬取这个页面的信息吧

import requests
from bs4 import BeautifulSoup
import json


def get_page():
    url = 'https://movie.douban.com/cinema/later/changsha/'
    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"}
    response = requests.get(url, headers, verify=False)
    text = response.text
    return text


def get_data(text):
    movies = []
    soup = BeautifulSoup(text, 'lxml')
    # class="item mod " class="item mod odd"有两个都是的。
    # 第一种思路，直接分开成两个来处理
    divList1 = soup.find_all('div', attrs={'class': "item mod "})
    divList2 = soup.find_all('div', attrs={'class': "item mod odd"})
    # 合并两个list
    divList = divList1+divList2
    '''
    <div class="item mod ">
<a class="thumb" href="https://movie.douban.com/subject/26394152/">
<img class="" src="https://img1.doubanio.com/view/photo/s_ratio_poster/public/p2541662397.jpg"/>
</a>
<div class="intro">
<h3>
<a class="" href="https://movie.douban.com/subject/26394152/">大黄蜂</a>
<span class="icon"></span>
</h3>
<li class="dt">01月04日</li>
<li class="dt">动作 / 科幻 / 冒险</li>
<li class="dt">美国</li>
</div>'''
    for div in divList:
        movie = {}
        thumb = div.find('img')['src']
        # print(thumb)
        # 获取所有a标签，其中第二个a标签里是电影名
        title = div.find_all('a')[1].text
        # print(title)
        # 有些电影里没有类型这列，所有要考虑下，不能直接赋值类型
        li_tag = div.find_all('li')
        if len(li_tag)==4:
            date = li_tag[0].text
            type = li_tag[1].text
            area = li_tag[2].text
            want = li_tag[3].text
        else:
            date = li_tag[0].text
            type = ''
            area = li_tag[1].text
            want = li_tag[2].text

        # print(date)
        movie['title'] = title
        movie['thumb'] = thumb
        movie['date'] = date
        movie['type'] = type
        movie['area'] = area
        movie['want'] = want
        movies.append(movie)
    return movies


def save_text(data):
    with open('douban2.json', 'w', encoding='utf-8') as fp:
        json.dump(data, fp, ensure_ascii=False)


if __name__ == '__main__':
    text = get_page()
    data = get_data(text)
    save_text(data)

两个li标签的区别

或者以后可以参考https://www.jianshu.com/p/c64fe2a20bc9，这个代码写的可以借鉴。