豆瓣电影

》》》基本方法

from urllib import request

response = request.urlopen("https://movie.douban.com/top250?start=25&filter=")
content = response.read().decode('utf-8')
print(content)

》》》代理服务器

from urllib import request
proxy_support = request.ProxyHandler({'http':'http://xx.xx.xx.xx:xx'})
opener = request.build_opener(proxy_support,request.HTTPHandler)
request.install_opener(opener)

content = request.urlopen('https://movie.douban.com/').read().decode('utf-8')
print(content)

》》》伪装成浏览器

postdata = parse.urlencode({})
headers ={

}

req = request.Request(
    url = "http//movie.douban.com    ",
    data = postdata,
)
headers = headers

》》》页面信息抓取

from urllib import request

class MoviesTop(object):
    def __init__(self):
        self.start = 0
        self.param = '&filter='
        self.headers = {'User-Agent':'Mozilla/5.0(Window NT 10.0;WOW64)'}

    def get_page(self):
        page_content = []
        try:
            while self.start <= 100:
                url = 'https://movie.douban.com/top250?start=' + str(self.start)
                req = request.Request(url,headers = self.headers)
                response = request.urlopen(req)
                page = response.read().decode('utf-8')
                # print(page)
                page_num = (self.start + 25)//25

                print('抓取'+str(page_num) + '页数据')
                self.start += 25
                page_content.append(page)
            return page_content
        except request.URLError as e:
            if hasattr(e,'reason'):
                print('抓取失败，原因是',e.reason)


    def main(self):
        print('开始抓取数据')
        self.get_page()
        print('数据抓取完毕')

a=MoviesTop()
a.main()

》》》提取相关信息

import re
# .*?
html_text =' <p class="quote"><span class="inq">希望让人自由。</span></p>'
reObj = re.compile('p class="quote"><span class="inq">(.*?)</span></p>')
print(reObj)
a = reObj.findall(html_text)
print(a)

》》》完整代码并将数据存入txt文件中

from urllib import request
import re

class MoviesTop(object):
    def __init__(self):
        self.start = 0
        self.param = '&filter='
        self.headers = {'User-Agent':'Mozilla/5.0(Window NT 10.0;WOW64)'}
        self.movies_list = []
        self.file_path = 'D:movies_spider2.xlsx'

    def get_page(self):
        try:
            # while self.start <= 75:
                url = 'https://movie.douban.com/top250?start=' + str(self.start)
                req = request.Request(url,headers = self.headers)
                response = request.urlopen(req)
                page = response.read().decode('utf-8')
                page_num = (self.start + 25)//25
                print('抓取'+str(page_num) + '页数据')
                self.start += 25
                return page
        except request.URLError as e:
            if hasattr(e,'reason'):
                print('抓取失败，原因是',e.reason)



    def get_movies_info(self):
        # pattern = re.compile(u'<span.*?class="title">(.*?)</span>.*?'
        #                      +u'<em.*?class="">(.*?)</em>.*?'
        #                      +u'<span.*?class="title">(.*?)</span>.*?',re.S)

        pattern = re.compile(u'<em.*?class="">(.*?)</em>.*?'
                             + u'<span.*?class="title">(.*?)</span>.*?'
                             + u'<span.*?class="title">&nbsp;/&nbsp;(.*?)</span>.*?'
                             +u'导演:(.*?)&nbsp;&nbsp;&nbsp;.*?'
                             +u'主演:(.*?)<br>.*?'
                             +u'<span>(.*?)人评价</span>.*?'
                             +u'<span.*?class="inq">(.*?)</span>.*?', re.S)

        # pattern = re.compile( u'<em.*?class="">(.*?)</em>.*?'
        #                      + u'<span.*?class="title">(.*?)</span>.*?'
        #                      + u'p class="quote"><span class="inq">(.*?)</span></p>', re.S)#排名，名称
        print('>>>',pattern)

        while self.start <50:
            page = self.get_page()
            # print(page)
            movies = re.findall(pattern,page)
            # print(movies)
            for movie in movies:
                print(movie)
                self.movies_list.append([movie[0],movie[1],movie[2],movie[3],movie[4],movie[5],movie[6]])
                print(movie[6])


    def write_text(self):
        print("开始写数据")
        file_top = open(self.file_path, 'w', encoding='utf-8')
        try:
            for movie in self.movies_list:
                file_top.write('电影排名: ' +  movie[0] + '
')
                file_top.write('电影名称：' +  movie[1] +'
')
                file_top.write('电影别名：' + movie[2] + '
')
                file_top.write('导演姓名：' + movie[3] + '
')
                file_top.write('主演姓名：' + movie[4] + '
')
                file_top.write('参评人数：' + movie[5] + '
')
                file_top.write('剪短影评：' + movie[6] + '
')

            print('写入成功')
        except Exception as e:
            print(e)
        finally:
            file_top.close()

    def main(self):

        print('开始抓取数据')
        self.get_movies_info()
        self.write_text()
        print('数据抓取完毕')


a = MoviesTop()
a.main()

 >>>完整代码并将数据存入数据库

from urllib import request
import re, MySQLdb


class MoviesTop(object):
    def __init__(self):
        self.start = 0
        self.param = '&filter='
        self.headers = {'User-Agent': 'Mozilla/5.0(Window NT 10.0;WOW64)'}
        self.movies_list = []
        # self.file_path = 'D:movies_spider2.txt'

    def get_page(self):
        try:
            # while self.start <= 75:
            url = 'https://movie.douban.com/top250?start=' + str(self.start)
            req = request.Request(url, headers=self.headers)
            response = request.urlopen(req)
            page = response.read().decode('utf-8')
            page_num = (self.start + 25) // 25
            print('抓取' + str(page_num) + '页数据')
            self.start += 25
            return page
        except request.URLError as e:
            if hasattr(e, 'reason'):
                print('抓取失败，原因是', e.reason)


def get_movies_info(self):
    # pattern = re.compile(u'<span.*?class="title">(.*?)</span>.*?'
    #                      +u'<em.*?class="">(.*?)</em>.*?'
    #                      +u'<span.*?class="title">(.*?)</span>.*?',re.S)

    pattern = re.compile(u'<em.*?class="">(.*?)</em>.*?'
                         + u'<span.*?class="title">(.*?)</span>.*?'
                         + u'<span.*?class="title">&nbsp;/&nbsp;(.*?)</span>.*?', re.S)
    # print('>>>',pattern)

    while self.start < 25:
        page = self.get_page()
        # print(page)
        movies = re.findall(pattern, page)
        print(movies[1])
        print(movies[1][2])
        for movie in movies:
            print(movie)
            self.movies_list.append([movie[0], movie[1], movie[2]])
            # print(self.movies_list)


def insert_into_sql(self):
    # conn=MySQLdb.connect(host='localhost',port='3306',user='root',password='123666',db='test')
    conn = MySQLdb.connect(host='localhost', port=3306, user='root', passwd='123666', db='test', charset='utf8')
    cur = conn.cursor()

    try:
        for movie in self.movies_list:
            a = movie[0]
            print(a)
            b = movie[1]
            print(b)
            c = movie[2]
            print(c)
            # insert_str = "insert into movies01 values( 2,'ghj', 'GHJ')"
            insert_str = "insert into m1 (moviesRank,MoviesName,OtherName) values('%s','%s','%s')" % (
                movie[0], movie[1], movie[2])
            # insert_sql =insert_str%()
            cur.execute(insert_str)
            # cur.close()
            conn.commit()
            # conn.close()
            print('>>>>')
    except Exception as e:
        print(e)


def main(self):
    print('开始抓取数据')
    self.get_movies_info()
    self.insert_into_sql()
    print('数据抓取完毕')


b = MoviesTop()
b.main()